Fix image build (opea-project#737)

Signed-off-by: chensuyue <[email protected]>
minmin-intel · Sep 27, 2024 · c4f9083 · c4f9083
1 parent 7134899
commit c4f9083
Show file tree

Hide file tree

Showing 16 changed files with 33 additions and 158 deletions.
diff --git a/.github/workflows/_comps-workflow.yml b/.github/workflows/_comps-workflow.yml
@@ -65,6 +65,7 @@ jobs:
           cd ${{ github.workspace }}
           if [[ $(grep -c "vllm-openvino:" ${docker_compose_yml}) != 0 ]]; then
               git clone https://github.com/vllm-project/vllm.git vllm-openvino
+              cd ./vllm-openvino && git checkout v0.6.1 && cd ../
           fi
 
       - name: Build Image

diff --git a/.github/workflows/docker/compose/retrievers-compose-cd.yaml b/.github/workflows/docker/compose/retrievers-compose-cd.yaml
@@ -6,7 +6,7 @@ services:
   retriever-pgvector:
     build:
       dockerfile: comps/retrievers/pgvector/langchain/Dockerfile
-    image: ${REGISTRY:-opea}/retriever-qdrant:${TAG:-latest}
+    image: ${REGISTRY:-opea}/retriever-pgvector:${TAG:-latest}
   retriever-pinecone:
     build:
       dockerfile: comps/retrievers/pinecone/langchain/Dockerfile

diff --git a/.github/workflows/manual-comps-test.yml b/.github/workflows/manual-comps-test.yml
@@ -7,7 +7,7 @@ on:
     inputs:
       services:
         default: "asr"
-        description: "List of services to test [agent,asr,chathistory,dataprep,embeddings,guardrails,llms,lvms,nginx,prompt_registry,reranks,retrievers,tts,web_retrievers]"
+        description: "List of services to test [agent,asr,chathistory,dataprep,embeddings,feedback_management,finetuning,guardrails,intent_detection,knowledgegraphs,llms,lvms,nginx,prompt_registry,ragas,reranks,retrievers,tts,vectorstores,web_retrievers]"
         required: true
         type: string
       build:
@@ -21,7 +21,7 @@ on:
         required: false
         type: boolean
       tag:
-        default: "comps"
+        default: "rc"
         description: "Tag to apply to images"
         required: true
         type: string

diff --git a/.github/workflows/manual-docker-publish.yml b/.github/workflows/manual-docker-publish.yml
@@ -7,7 +7,7 @@ on:
     inputs:
       services:
         default: ""
-        description: "List of services to test [agent,asr,chathistory,dataprep,embeddings,guardrails,llms,lvms,nginx,prompt_registry,ragas,reranks,retrievers,tts,vectorstores,web_retrievers]"
+        description: "List of services to test [agent,asr,chathistory,dataprep,embeddings,feedback_management,finetuning,guardrails,intent_detection,knowledgegraphs,llms,lvms,nginx,prompt_registry,ragas,reranks,retrievers,tts,vectorstores,web_retrievers]"
         required: false
         type: string
       images:
@@ -16,15 +16,20 @@ on:
         required: false
         type: string
       tag:
-        default: "v0.9"
+        default: "rc"
         description: "Tag to publish"
         required: true
         type: string
       publish_tags:
-        default: "latest,v0.9"
+        default: "latest,1.x"
         description: "Tag list apply to publish images"
         required: false
         type: string
+      mode:
+        default: "CD"
+        description: "Whether the test range is CI or CD"
+        required: false
+        type: string
 
 permissions: read-all
 jobs:

diff --git a/.github/workflows/manual-image-build.yml b/.github/workflows/manual-image-build.yml
diff --git a/.github/workflows/schedule-image-build.yml b/.github/workflows/schedule-image-build.yml
diff --git a/comps/dataprep/neo4j/langchain/Dockerfile b/comps/dataprep/neo4j/langchain/Dockerfile
@@ -29,7 +29,7 @@ ENV PYTHONPATH=$PYTHONPATH:/home/user
 
 USER root
 
-RUN mkdir -p /home/user/comps/dataprep/qdrant/langchain/uploaded_files && chown -R user /home/user/comps/dataprep/neo4j/langchain/uploaded_files
+RUN mkdir -p /home/user/comps/dataprep/neo4j/langchain/uploaded_files && chown -R user /home/user/comps/dataprep/neo4j/langchain/uploaded_files
 
 USER user
 

diff --git a/comps/llms/text-generation/vllm/langchain/dependency/build_docker_vllm_openvino.sh b/comps/llms/text-generation/vllm/langchain/dependency/build_docker_vllm_openvino.sh
@@ -5,6 +5,6 @@
 
 BASEDIR="$( cd "$( dirname "$0" )" && pwd )"
 git clone https://github.com/vllm-project/vllm.git vllm
-cd ./vllm/
+cd ./vllm/ && git checkout v0.6.1
 docker build -t vllm:openvino -f Dockerfile.openvino . --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy
 cd $BASEDIR && rm -rf vllm
diff --git a/comps/llms/text-generation/vllm/llama_index/dependency/build_docker_vllm_openvino.sh b/comps/llms/text-generation/vllm/llama_index/dependency/build_docker_vllm_openvino.sh
@@ -5,6 +5,6 @@
 
 BASEDIR="$( cd "$( dirname "$0" )" && pwd )"
 git clone https://github.com/vllm-project/vllm.git vllm
-cd ./vllm/
+cd ./vllm/ && git checkout v0.6.1
 docker build -t vllm:openvino -f Dockerfile.openvino . --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy
 cd $BASEDIR && rm -rf vllm
diff --git a/comps/llms/utils/lm-eval/Dockerfile b/comps/llms/utils/lm-eval/Dockerfile
@@ -23,12 +23,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
 
 USER user
 
-ARG REPO_COMPS=https://github.com/opea-project/GenAIComps.git
-ARG BRANCH=main
-RUN git clone --single-branch --branch=${BRANCH} ${REPO_COMPS} /home/user/GenAIComps/ && \
-    cd /home/user/GenAIComps/ && python3 setup.py install && \
-    pip install --no-cache-dir -r /home/user/GenAIComps/comps/llms/utils/lm-eval/requirements.txt
+COPY comps /home/user/comps
+
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r /home/user/comps/llms/utils/lm-eval/requirements.txt
+
+ENV PYTHONPATH=$PYTHONPATH:/home/user
+
+WORKDIR /home/user/comps/llms/utils/lm-eval
 
-WORKDIR /home/user/GenAIComps/comps/llms/utils/lm-eval/
 
 ENTRYPOINT ["python3", "self_hosted_hf.py"]
diff --git a/comps/llms/utils/lm-eval/requirements.txt b/comps/llms/utils/lm-eval/requirements.txt
@@ -1,4 +1,4 @@
-git+https://github.com/bigcode-project/bigcode-evaluation-harness.git@a1b4a7949a24c8e3ef0d05a01097b2d14ffba56e
-git+https://github.com/opea-project/GenAIEval.git
-lm-eval==0.4.2
+bigcode-eval@git+https://github.com/bigcode-project/bigcode-evaluation-harness.git
+lm-eval==0.4.3
+opea-eval
 pydantic==2.7.2
diff --git a/comps/vectorstores/pathway/Dockerfile b/comps/vectorstores/pathway/Dockerfile
@@ -15,12 +15,12 @@ RUN apt-get update && apt-get install -y \
 
 WORKDIR /app
 
-COPY requirements.txt /app/
+COPY comps/vectorstores/pathway/requirements.txt /app/
 
 RUN if [ ${ARCH} = "cpu" ]; then pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu; fi && \
     pip install --no-cache-dir -r requirements.txt
 
-COPY vectorstore_pathway.py /app/
+COPY comps/vectorstores/pathway/vectorstore_pathway.py /app/
 
 
 CMD ["python", "vectorstore_pathway.py"]

diff --git a/comps/vectorstores/pathway/README.md b/comps/vectorstores/pathway/README.md
@@ -57,9 +57,7 @@ For more information, see the relevant Pathway docs:
 Build the Docker and run the Pathway Vector Store:
 
 ```bash
-cd comps/vectorstores/langchain/pathway/docker
-
-docker build --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -t opea/vectorstore-pathway:latest .
+docker build --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -t opea/vectorstore-pathway:latest -f comps/vectorstores/pathway/Dockerfile .
 
 # with locally loaded model, you may add `EMBED_MODEL` env variable to configure the model.
 docker run -e PATHWAY_HOST=${PATHWAY_HOST} -e PATHWAY_PORT=${PATHWAY_PORT} -e http_proxy=$http_proxy -e https_proxy=$https_proxy -v ./data:/app/data -p ${PATHWAY_PORT}:${PATHWAY_PORT} opea/vectorstore-pathway:latest

diff --git a/tests/test_image2video.sh → tests/image2video/test_image2video.sh b/tests/test_image2video.sh → tests/image2video/test_image2video.sh
diff --git a/tests/llms/test_llms_text-generation_vllm_langchain_openvino.sh b/tests/llms/test_llms_text-generation_vllm_langchain_openvino.sh
@@ -15,7 +15,8 @@ CONTAINER_NAME="test-comps-vllm-openvino-container"
 function build_container() {
     cd $WORKPATH
     git clone https://github.com/vllm-project/vllm.git vllm-openvino
-    cd ./vllm-openvino/
+    cd ./vllm-openvino/ && git checkout v0.6.1 # something wrong with main branch image build
+
     docker build --no-cache -t $DOCKER_IMAGE \
       -f Dockerfile.openvino \
       . \

diff --git a/tests/vectorstores/test_vectorstores_pathway.sh b/tests/vectorstores/test_vectorstores_pathway.sh
@@ -10,9 +10,7 @@ ip_address=$(hostname -I | awk '{print $1}')
 function build_docker_images() {
     cd $WORKPATH
 
-    cd comps/vectorstores/pathway
-
-    docker build --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -t opea/vectorstore-pathway:comps .
+    docker build --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -t opea/vectorstore-pathway:comps -f comps/vectorstores/pathway/Dockerfile .
 
     cd $WORKPATH
 
@@ -47,7 +45,7 @@ function start_service() {
     export PATHWAY_HOST="0.0.0.0"
     export PATHWAY_PORT=5437
 
-    docker run -d --name="test-comps-vectorstore-pathway-ms" -e PATHWAY_HOST=${PATHWAY_HOST} -e PATHWAY_PORT=${PATHWAY_PORT} -e TEI_EMBEDDING_ENDPOINT=${TEI_EMBEDDING_ENDPOINT} -e http_proxy=$http_proxy -e https_proxy=$https_proxy -v $WORKPATH/comps/vectorstores/langchain/pathway/README.md:/app/data/README.md -p ${PATHWAY_PORT}:${PATHWAY_PORT} --network="host" opea/vectorstore-pathway:comps
+    docker run -d --name="test-comps-vectorstore-pathway-ms" -e PATHWAY_HOST=${PATHWAY_HOST} -e PATHWAY_PORT=${PATHWAY_PORT} -e TEI_EMBEDDING_ENDPOINT=${TEI_EMBEDDING_ENDPOINT} -e http_proxy=$http_proxy -e https_proxy=$https_proxy -v $WORKPATH/comps/vectorstores/pathway/README.md:/app/data/README.md -p ${PATHWAY_PORT}:${PATHWAY_PORT} --network="host" opea/vectorstore-pathway:comps
 
     sleep 45s