Merge branch 'add_modeltest' of https://github.com/opea-project/GenAI…

…Eval into add_modeltest

Signed-off-by: Wenxin Zhang <[email protected]>

.github/workflows/model_test.yml

@@ -37,7 +37,7 @@ jobs:
       matrix:
         include:
           - modelName: "EleutherAI/gpt-j-6B"
-            task: "hellaswag"
+            tasks: "hellaswag"
             device: "cpu"
       fail-fast: true
 
@@ -63,7 +63,7 @@ jobs:
           fi
           docker run -dit --disable-content-trust --privileged --name=${{ env.CONTAINER_NAME }} -v /dev/shm:/dev/shm \
           -v ${{ github.workspace }}:/GenAIEval \
-          ${{ env.REPO_NAME }}:${{ env.DOCKER_TAG }}
+          ${{ env.DOCKER_NAME }}:${{ env.DOCKER_TAG }}
 
       - name: Binary build
         run: |
@@ -92,15 +92,15 @@ jobs:
         run: |
             docker exec ${{ env.CONTAINER_NAME }} \
             bash -c "cd /GenAIEval/.github/workflows/scripts/models \
-            && bash model_test.sh --model=${{ matrix.modelName }} --device=${{ matrix.device }} --tasks=${{ matrix.task }}"
+            && bash model_test.sh --model=${{ matrix.modelName }} --device=${{ matrix.device }} --tasks=${{ matrix.tasks }}"
 
       - name: Collect Log
         run: |
             docker exec ${{ env.CONTAINER_NAME }} \
             bash -c "cd /GenAIEval/.github/workflows/scripts/models \
             && bash -x collect_log.sh --model=${{ matrix.modelName }} \
              --device=${{ matrix.device }} \
-             --task=${{ matrix.task }}
+             --tasks=${{ matrix.tasks }}
 
       - name: Publish pipeline artifact
         uses: actions/upload-artifact@v4

.github/workflows/scripts/codeScan/trellix.sh

@@ -57,4 +57,4 @@ if [[ $(grep "Possibly Infected" ${log_dir}/trellix.log | sed 's/[^0-9]//g') !=
 fi
 
 $BOLD_PURPLE && echo "Congratulations, Trellix Scan passed!" && $LIGHT_PURPLE && echo " You can click on the artifact button to see the log details." && $RESET
-exit 0
+exit 0

.github/workflows/scripts/models/collect_log.sh

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 set -eo pipefail
-source /GenAIEval/.github/workflows/script/change_color.sh
+source /GenAIEval/.github/workflows/scripts/change_color
 WORKSPACE="/GenAIEval"
 # get parameters
 PATTERN='[-a-zA-Z0-9_]*='
@@ -25,21 +25,21 @@ for i in "$@"; do
             device=`echo $i | sed "s/${PATTERN}//"`;;
         --model=*)
             model=`echo $i | sed "s/${PATTERN}//"`;;
-        --task=*)
-            task=`echo $i | sed "s/${PATTERN}//"`;;
+        --tasks=*)
+            tasks=`echo $i | sed "s/${PATTERN}//"`;;
         *)
             echo "Parameter $i not recognized."; exit 1;;
     esac
 done
 
-output_file="/GenAIEval/${device}/${model}/${device}-${model}-${task}.log"
+output_file="/GenAIEval/${device}/${model}/${device}-${model}-${tasks}.log"
 $BOLD_YELLOW && echo "-------- Collect logs --------" && $RESET
 
 echo "working in"
 pwd
 if [[ ! -f ${output_file} ]]; then
-    echo "${device};${model};${task};;${logfile}" >> ${WORKSPACE}/summary.log
+    echo "${device};${model};${tasks};;${logfile}" >> ${WORKSPACE}/summary.log
 else
     acc=$(grep -Po "Accuracy .* is:\\s+(\\d+(\\.\\d+)?)" ${acc_log_name} | head -n 1 | sed 's/.*://;s/[^0-9.]//g')
-    echo "${device};${model};${task};${acc};${logfile}" >> ${WORKSPACE}/summary.log
+    echo "${device};${model};${tasks};${acc};${logfile}" >> ${WORKSPACE}/summary.log
 fi

.github/workflows/scripts/models/model_test.sh

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 set -eo pipefail
-source /GenAIEval/.github/workflows/script/change_color.sh
+source /GenAIEval/.github/workflows/scripts/change_color
 
 # get parameters
 PATTERN='[-a-zA-Z0-9_]*='
@@ -25,8 +25,8 @@ for i in "$@"; do
             device=`echo $i | sed "s/${PATTERN}//"`;;
         --model=*)
             model=`echo $i | sed "s/${PATTERN}//"`;;
-        --task=*)
-            task=`echo $i | sed "s/${PATTERN}//"`;;
+        --tasks=*)
+            tasks=`echo $i | sed "s/${PATTERN}//"`;;
         *)
             echo "Parameter $i not recognized."; exit 1;;
     esac
@@ -59,11 +59,11 @@ function prepare() {
 function run_benchmark() {
     cd ${working_dir}
     pip install --upgrade-strategy eager optimum[habana]
-    overall_log="${log_dir}/${device}-${model}-${task}.log"
+    overall_log="${log_dir}/${device}-${model}-${tasks}.log"
     python main.py \
         --model hf \
         --model_args pretrained=${model} \
-        --tasks ${task} \
+        --tasks ${tasks} \
         --device ${device} \
         --batch_size 8
         2>&1 | tee ${overall_log}

.github/workflows/scripts/unittest/calc_coverage.sh

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-source ../../change_color.sh
+source ../../change_color
 LOG_DIR=$1
 coverage_compare="${LOG_DIR}/coverage_compare.html"
 coverage_log_pr="${LOG_DIR}/UnitTestPR-test/coverage_pr"