From 5738a2e6e0639cade7a962af14d1fd91b3816b4f Mon Sep 17 00:00:00 2001 From: priscilla-pan Date: Thu, 13 Aug 2020 06:06:04 +0000 Subject: [PATCH] Support benchmark about model optimization (#12) Closes #6 Co-authored-by: priscilla-pan --- benchmark/README.md | 44 +++++++++++++++++++ .../client_script/resnet50_client_script.sh | 4 ++ .../resnet50_pruned_client_script.sh | 4 ++ .../resnet50_pruned_serving_script.sh | 3 ++ .../serving_script/resnet50_serving_script.sh | 3 ++ .../resnet50_pruned_serving_model.json | 22 ++++++++++ .../resnet50_serving_model.json | 22 ++++++++++ benchmark/tools/resnet50/automatic_test.sh | 15 +++++++ .../tools/resnet50/automatic_test_pruned.sh | 18 ++++++++ 9 files changed, 135 insertions(+) create mode 100644 benchmark/README.md create mode 100644 benchmark/tests/client_script/resnet50_client_script.sh create mode 100644 benchmark/tests/client_script/resnet50_pruned_client_script.sh create mode 100644 benchmark/tests/serving_script/resnet50_pruned_serving_script.sh create mode 100644 benchmark/tests/serving_script/resnet50_serving_script.sh create mode 100644 benchmark/tests/test_model/resnet50_keras/resnet50_pruned_serving_model.json create mode 100644 benchmark/tests/test_model/resnet50_keras/resnet50_serving_model.json create mode 100755 benchmark/tools/resnet50/automatic_test.sh create mode 100755 benchmark/tools/resnet50/automatic_test_pruned.sh diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..fd65505 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,44 @@ +# About the benchmark + +The benchmark is used to test the adlik serving performance of different models which are optimized by model_optimizer. + +## Test the runtime performance + +Take ResNet-50 as an example to illustrate how to test the model performance + +1. clone model_optimizer and change the working directory into the source directory + + ```sh + git clone https://github.com/Adlik/model_optimizer.git + cd model_optimizer + ``` + +2. make a model directory and put resnet50.h5 in it + + ```sh + mkdir -p benchmark/tests/test_model/resnet50_keras/model + ``` + +3. change the working directory into benchmark tools directory, execute the shell script automatic_test.sh, which can +auto test the performance of ResNet50 + + ```sh + cd benchmark/tools/resnet50 + ./automatic_test.sh + ``` + +4. you can also test the performance of optimized model of ResnNet-50. Put the resnet50_pruned.h5 under model directory + and execute the shell script automatic_test_pruned.sh + +## The test result of the model in keras-tfGPU2.1 + +| | speed of client (pictures/sec) | speed of serving engine (pictures/sec) | tail latency of one picture (sec) | +| ---------------- | :----------------------------: | :------------------------------------: | :-------------------------------: | +| ResNet-50 | 184.872 | 480.882 | 0.00333 | +| ResNet-50-L1-0.3 | 191.531 | 518.280 | 0.00329 | + +> Note +> +> > i. ResNet-50 is the baseline of training +> > +> > ii. ResNet-50-L1-0.3 denotes ResNet-50 model use L1 norm filter pruning, pruning rate is 30% diff --git a/benchmark/tests/client_script/resnet50_client_script.sh b/benchmark/tests/client_script/resnet50_client_script.sh new file mode 100644 index 0000000..2f71b6f --- /dev/null +++ b/benchmark/tests/client_script/resnet50_client_script.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +python3 /home/john/Adlik/benchmark/tests/client/$CLIENT_INFERENCE_SCRIPT --model-name=resnet50 --batch-size=100 /home/john/Adlik/benchmark/tests/data/$IMAGE_FILENAME && \ +mv /home/john/Adlik/client_time.log /home/john/log/client_time.log && \ +mv /home/john/Adlik/serving_time.log /home/john/log/serving_time.log diff --git a/benchmark/tests/client_script/resnet50_pruned_client_script.sh b/benchmark/tests/client_script/resnet50_pruned_client_script.sh new file mode 100644 index 0000000..7486b5d --- /dev/null +++ b/benchmark/tests/client_script/resnet50_pruned_client_script.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +python3 /home/john/Adlik/benchmark/tests/client/$CLIENT_INFERENCE_SCRIPT --model-name=resnet50_pruned --batch-size=100 /home/john/Adlik/benchmark/tests/data/$IMAGE_FILENAME && \ +mv /home/john/Adlik/client_time.log /home/john/log/client_time.log && \ +mv /home/john/Adlik/serving_time.log /home/john/log/serving_time.log diff --git a/benchmark/tests/serving_script/resnet50_pruned_serving_script.sh b/benchmark/tests/serving_script/resnet50_pruned_serving_script.sh new file mode 100644 index 0000000..51c06de --- /dev/null +++ b/benchmark/tests/serving_script/resnet50_pruned_serving_script.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +cd /home/john/Adlik/bazel-bin/adlik_serving && \ +./adlik_serving --model_base_path=/home/john/Adlik/model/model_repos_imagenet_resnet50_pruned_keras --grpc_port=8500 --http_port=8501 >> /home/john/Adlik/serving_time.log 2>&1 diff --git a/benchmark/tests/serving_script/resnet50_serving_script.sh b/benchmark/tests/serving_script/resnet50_serving_script.sh new file mode 100644 index 0000000..6786717 --- /dev/null +++ b/benchmark/tests/serving_script/resnet50_serving_script.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +cd /home/john/Adlik/bazel-bin/adlik_serving && \ +./adlik_serving --model_base_path=/home/john/Adlik/model/model_repos_imagenet_resnet50_keras --grpc_port=8500 --http_port=8501 >> /home/john/Adlik/serving_time.log 2>&1 diff --git a/benchmark/tests/test_model/resnet50_keras/resnet50_pruned_serving_model.json b/benchmark/tests/test_model/resnet50_keras/resnet50_pruned_serving_model.json new file mode 100644 index 0000000..7d6ee99 --- /dev/null +++ b/benchmark/tests/test_model/resnet50_keras/resnet50_pruned_serving_model.json @@ -0,0 +1,22 @@ +{ + "serving_type": "tf", + "input_model": "model/resnet50_pruned.h5", + "export_path": "model_repos_imagenet_resnet50_pruned_keras", + "input_layer_names": [ + "input" + ], + "output_layer_names": [ + "dense1" + ], + "input_formats": [ + "channels_last" + ], + "input_signatures": [ + "image" + ], + "output_signatures": [ + "label" + ], + "model_name": "resnet50_pruned", + "max_batch_size": 256 +} diff --git a/benchmark/tests/test_model/resnet50_keras/resnet50_serving_model.json b/benchmark/tests/test_model/resnet50_keras/resnet50_serving_model.json new file mode 100644 index 0000000..bf8778a --- /dev/null +++ b/benchmark/tests/test_model/resnet50_keras/resnet50_serving_model.json @@ -0,0 +1,22 @@ +{ + "serving_type": "tf", + "input_model": "model/resnet50.h5", + "export_path": "model_repos_imagenet_resnet50_keras", + "input_layer_names": [ + "input" + ], + "output_layer_names": [ + "dense1" + ], + "input_formats": [ + "channels_last" + ], + "input_signatures": [ + "image" + ], + "output_signatures": [ + "label" + ], + "model_name": "resnet50", + "max_batch_size": 256 +} diff --git a/benchmark/tools/resnet50/automatic_test.sh b/benchmark/tools/resnet50/automatic_test.sh new file mode 100755 index 0000000..4c37a30 --- /dev/null +++ b/benchmark/tools/resnet50/automatic_test.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +current_path=$(cd `dirname $0`; pwd) +echo "current_path: "$current_path +base_path=$current_path/../../../.. +echo "base_path: "$base_path +cd $base_path +git clone https://github.com/Adlik/Adlik.git +optimizer_benchmark_path=$base_path/model_optimizer/benchmark +adlik_benchmark_path=$base_path/Adlik/benchmark +adlik_path=$base_path/Adlik +cp -r $optimizer_benchmark_path/tests $adlik_benchmark_path + +cd $adlik_path +echo "*****************start to make benchmark image and test****************************" +python3 benchmark/src/automatic_test.py -d benchmark/tests/docker_test/tensorflow_gpu.Dockerfile -s tensorflow_gpu -b . -a . -m resnet50 -c benchmark/tests/client_script/resnet50_client_script.sh -ss benchmark/tests/serving_script/resnet50_serving_script.sh -l /home/pjy/adlik/Adlik/benchmark/log -tm benchmark/tests/test_model/resnet50_keras -cis imagenet_client.py -i imagenet.JPEG -cs benchmark/tests/compile_script/compile_script.sh -sj resnet50_serving_model.json -gl 0 diff --git a/benchmark/tools/resnet50/automatic_test_pruned.sh b/benchmark/tools/resnet50/automatic_test_pruned.sh new file mode 100755 index 0000000..23df97d --- /dev/null +++ b/benchmark/tools/resnet50/automatic_test_pruned.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +current_path=$(cd `dirname $0`; pwd) +echo "current_path: "$current_path +base_path=$current_path/../../../.. +echo "base_path: "$base_path +cd $base_path + +git clone https://github.com/Adlik/Adlik.git +base_path=$(cd `dirname $0`; pwd) +echo "base_path: "$base_path +optimizer_benchmark_path=$base_path/model_optimizer/benchmark +adlik_benchmark_path=$base_path/Adlik/benchmark +adlik_path=$base_path/Adlik +cp -r $optimizer_benchmark_path/tests $adlik_benchmark_path + +cd $adlik_path +echo "*****************start to make benchmark image and test****************************" +python3 benchmark/src/automatic_test.py -d benchmark/tests/docker_test/tensorflow_gpu.Dockerfile -s tensorflow_gpu -b . -a . -m resnet50_pruned -c benchmark/tests/client_script/resnet50_pruned_client_script.sh -ss benchmark/tests/serving_script/resnet50_pruned_serving_script.sh -l /home/pjy/adlik/Adlik/benchmark/log -tm benchmark/tests/test_model/resnet50_keras -cis imagenet_client.py -i imagenet.JPEG -cs benchmark/tests/compile_script/compile_script.sh -sj resnet50_pruned_serving_model.json -gl 0