Build docker (#515)

* Updated Docker image
alibaba · Jan 16, 2025 · 8e2def1 · 8e2def1
1 parent 4b0b1f5
commit 8e2def1
Show file tree

Hide file tree

Showing 8 changed files with 220 additions and 10 deletions.
diff --git a/docker/Dockerfile_tf112 b/docker/Dockerfile_tf112
@@ -0,0 +1,74 @@
+#FROM tensorflow/tensorflow:1.12.0
+FROM my_tensorflow_base:1.12.0-py2
+
+COPY docker/sources_18.04.list /etc/apt/sources.list
+
+# necessary for later commands to take effect
+RUN md5sum /etc/apt/sources.list \
+    && apt-get update \
+    && apt-get install apt-utils inetutils-ping wget curl telnet vim strace libpq-dev curl libsasl2-dev gcc g++ unzip openjdk-8-jdk -y \
+    && apt-get install build-essential cython -y \
+    && pip install cython \
+    && pip install setuptools_scm
+# 检查 Cython 是否安装成功
+RUN python -c "import Cython"
+RUN pip --version
+
+RUN mkdir /EasyRec
+COPY requirements /EasyRec/requirements
+COPY requirements.txt /EasyRec/
+COPY easy_rec /EasyRec/easy_rec/
+COPY setup.cfg  /EasyRec/
+COPY setup.py /EasyRec/
+COPY MANIFEST.in /EasyRec/
+COPY README.md /EasyRec/
+COPY scripts /EasyRec/scripts
+
+RUN curl "http://easyrec.oss-cn-beijing.aliyuncs.com/tools/odpscmd_public_0.45.0.zip" -o /EasyRec/odpscmd_public.zip
+RUN mkdir /usr/local/odps_clt/ && cd /usr/local/odps_clt/ && unzip /EasyRec/odpscmd_public.zip
+RUN ln -s /usr/local/odps_clt/bin/odpscmd /usr/local/bin/odpscmd
+
+RUN pip install pystack-debugger idna kafka-python -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
+# 升级pip
+RUN pip install --upgrade pip setuptools wheel
+
+# 安装 setuptools-rust 和 rustc
+RUN pip install setuptools-rust
+RUN pip install tensorflow_probability==0.5.0
+RUN apt-get update && apt-get install -y rustc
+RUN apt-get update && \
+    apt-get install -y rustc && \
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \
+    . $HOME/.cargo/env
+# 安装 cryptography
+RUN pip install cryptography
+# 安装基础工具链与依赖项
+RUN apt-get update && \
+    apt-get install -y build-essential libssl-dev libffi-dev python-dev && \
+    apt-get install -y rustc cargo cmake curl
+
+# 设置国内的 Rust 镜像源
+RUN echo '[source.crates-io]\n' > $HOME/.cargo/config
+RUN echo 'replace-with = "ustc"' >> $HOME/.cargo/config
+RUN echo '[source.ustc]\n' >> $HOME/.cargo/config
+RUN echo 'registry = "https://mirrors.ustc.edu.cn/crates.io-index"' >> $HOME/.cargo/config
+
+# 确保 curl 支持 HTTP2
+RUN curl -V
+
+# 显示安装好的工具链版本，确保已正确安装
+RUN rustc --version
+RUN cargo --version
+RUN cmake --version
+
+RUN pip install -r /EasyRec/requirements/runtime.txt   -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
+RUN pip install -r /EasyRec/requirements/extra.txt   -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
+
+RUN pip install --user -U https://tfsmoke1.oss-cn-zhangjiakou.aliyuncs.com/tunnel_paiio/common_io/py2/common_io-0.1.0-cp27-cp27mu-linux_x86_64.whl
+RUN pip install graphlearn
+
+RUN cd /EasyRec && python setup.py install
+RUN rm -rf /EasyRec
+RUN python -c "import easy_rec; import pyhive; import datahub; import kafka"
+
+COPY docker/hadoop_env.sh /opt/hadoop_env.sh
diff --git a/docker/Dockerfile_tf115 b/docker/Dockerfile_tf115
@@ -0,0 +1,36 @@
+FROM datascience-registry.cn-beijing.cr.aliyuncs.com/tensorflow/tensorflow:1.15.5
+
+COPY docker/sources_18.04.list /etc/apt/sources.list
+
+# necessary for later commands to take effect
+RUN md5sum /etc/apt/sources.list
+
+RUN apt-get update
+RUN apt-get install apt-utils inetutils-ping wget curl telnet vim strace libpq-dev curl libsasl2-dev gcc g++ unzip openjdk-8-jdk -y
+
+RUN mkdir /EasyRec
+COPY requirements /EasyRec/requirements
+COPY requirements.txt /EasyRec/
+COPY easy_rec /EasyRec/easy_rec/
+COPY setup.cfg  /EasyRec/
+COPY setup.py /EasyRec/
+COPY MANIFEST.in /EasyRec/
+COPY README.md /EasyRec/
+COPY scripts /EasyRec/scripts
+
+RUN curl "http://easyrec.oss-cn-beijing.aliyuncs.com/tools/odpscmd_public_0.45.0.zip" -o /EasyRec/odpscmd_public.zip
+RUN mkdir /usr/local/odps_clt/ && cd /usr/local/odps_clt/ && unzip /EasyRec/odpscmd_public.zip
+RUN ln -s /usr/local/odps_clt/bin/odpscmd /usr/local/bin/odpscmd
+RUN pip3 install --upgrade pip
+RUN pip3 install pystack-debugger idna kafka-python -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
+RUN pip3 install -r /EasyRec/requirements/runtime.txt
+RUN pip3 install -r /EasyRec/requirements/extra.txt   -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
+RUN pip3 install http://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/graphlearn-1.1.0-cp36-cp36m-linux_x86_64.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
+RUN pip3 install http://easyrec.oss-cn-beijing.aliyuncs.com/releases/pai_automl-0.0.1rc1-py3-none-any.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
+RUN pip3 install http://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/common_io-0.3.0-cp36-cp36m-linux_x86_64.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
+RUN pip3 install tensorflow_probability==0.8
+RUN cd /EasyRec && pip install .
+RUN rm -rf /EasyRec
+RUN python -c "import easy_rec; easy_rec.help(); import pyhive; import datahub; import kafka"
+
+COPY docker/hadoop_env.sh /opt/hadoop_env.sh
diff --git a/docker/Dockerfile_tf212 b/docker/Dockerfile_tf212
@@ -0,0 +1,36 @@
+FROM tensorflow/tensorflow:2.12.0
+COPY docker/sources_20.04.list /etc/apt/sources.list
+
+# necessary for later commands to take effect
+RUN md5sum /etc/apt/sources.list
+
+RUN apt-get update
+RUN apt-get install apt-utils inetutils-ping wget curl telnet vim strace libpq-dev curl libsasl2-dev gcc g++ unzip openjdk-8-jdk -y
+
+RUN mkdir /EasyRec
+COPY requirements /EasyRec/requirements
+COPY requirements.txt /EasyRec/
+COPY easy_rec /EasyRec/easy_rec/
+COPY setup.cfg  /EasyRec/
+COPY setup.py /EasyRec/
+COPY MANIFEST.in /EasyRec/
+COPY README.md /EasyRec/
+COPY scripts /EasyRec/scripts
+
+RUN curl "http://easyrec.oss-cn-beijing.aliyuncs.com/tools/odpscmd_public_0.45.0.zip" -o /EasyRec/odpscmd_public.zip
+RUN mkdir /usr/local/odps_clt/ && cd /usr/local/odps_clt/ && unzip /EasyRec/odpscmd_public.zip
+RUN ln -s /usr/local/odps_clt/bin/odpscmd /usr/local/bin/odpscmd
+
+RUN pip3 install pystack-debugger idna kafka-python -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
+RUN pip3 install -r /EasyRec/requirements/runtime.txt   -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
+RUN pip3 install -r /EasyRec/requirements/extra.txt   -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
+RUN pip3 install https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/graphlearn-1.1.0-cp38-cp38-linux_x86_64.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
+# RUN pip3 install http://easyrec.oss-cn-beijing.aliyuncs.com/releases/pai_automl-0.0.1rc1-py3-none-any.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
+RUN pip3 install tensorflow_probability==0.20.0
+#RUN pip3 install encodings
+RUN pip3 install https://dlc-task.oss-cn-hangzhou.aliyuncs.com/whl/common_io-0.4.1%2Btunnel-py2.py3-none-any.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
+RUN cd /EasyRec && python setup.py install
+RUN rm -rf /EasyRec
+# RUN python -c "import easy_rec; easy_rec.help(); import pyhive; import datahub; import kafka"
+
+COPY docker/hadoop_env.sh /opt/hadoop_env.sh
diff --git a/examples/readme.md b/examples/readme.md
@@ -36,14 +36,14 @@ cd EasyRec
 
 -- Docker环境可选
 (1) `python=3.6.9` + `tenserflow=1.15.5`
-docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.7.4
-docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.7.4
+docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.8.5
+docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.8.5
 docker exec -it <CONTAINER_ID> bash
 
 
-(2) `python=3.8.10` + `tenserflow=2.10.0`
-docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-0.7.4
-docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-0.7.4
+(2) `python=3.8.10` + `tenserflow=2.12.0`
+docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.12-0.8.5
+docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.12-0.8.5
 
 docker exec -it <CONTAINER_ID> bash
 ```
@@ -56,12 +56,12 @@ cd EasyRec
 
 -- Docker环境可选
 (1) `python=3.6.9` + `tenserflow=1.15.5`
-bash scripts/build_docker.sh
+bash scripts/build_docker_tf115.sh
 sudo docker run -td --network host -v /local_path:/docker_path mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-<easyrec_version>
 
-(2) `python=3.8.10` + `tenserflow=2.10.0`
-bash scripts/build_docker_tf210.sh
-sudo docker run -td --network host -v /local_path:/docker_path mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-<easyrec_version>
+(2) `python=3.8.10` + `tenserflow=2.12.0`
+bash scripts/build_docker_tf212.sh
+sudo docker run -td --network host -v /local_path:/docker_path mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.12-<easyrec_version>
 
 sudo docker exec -it <CONTAINER_ID> bash
 ```

diff --git a/requirements/runtime.txt b/requirements/runtime.txt
@@ -1,4 +1,5 @@
-eas_prediction
+eas_prediction == 0.24; python_version < '3.0'
+eas_prediction; python_version >= '3.0'
 future
 matplotlib
 numpy <= 1.23

diff --git a/scripts/build_docker_tf112.sh b/scripts/build_docker_tf112.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+bash scripts/gen_proto.sh
+if [ $? -ne 0 ]
+then
+  echo "gen proto failed"
+  exit 1
+fi
+
+version=`grep "__version__" easy_rec/version.py | awk '{ if($1 == "__version__") print $NF}'`
+# strip "'"
+version=${version//\'/}
+echo "EasyRec Version: $version"
+
+if [ -z "$version" ]
+then
+  echo "Failed to get EasyRec version"
+  exit 1
+fi
+
+sudo docker build --network=host . -f docker/Dockerfile_tf112 -t  mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py27-tf1.12-${version}
diff --git a/scripts/build_docker_tf115.sh b/scripts/build_docker_tf115.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+bash scripts/gen_proto.sh
+if [ $? -ne 0 ]
+then
+  echo "gen proto failed"
+  exit 1
+fi
+
+version=`grep "__version__" easy_rec/version.py | awk '{ if($1 == "__version__") print $NF}'`
+# strip "'"
+version=${version//\'/}
+echo "EasyRec Version: $version"
+
+if [ -z "$version" ]
+then
+  echo "Failed to get EasyRec version"
+  exit 1
+fi
+
+sudo docker build --network=host . -f docker/Dockerfile_tf115 -t  mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-${version}
diff --git a/scripts/build_docker_tf212.sh b/scripts/build_docker_tf212.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+bash scripts/gen_proto.sh
+if [ $? -ne 0 ]
+then
+  echo "gen proto failed"
+  exit 1
+fi
+
+version=`grep "__version__" easy_rec/version.py | awk '{ if($1 == "__version__") print $NF}'`
+# strip "'"
+version=${version//\'/}
+echo "EasyRec Version: $version"
+
+if [ -z "$version" ]
+then
+  echo "Failed to get EasyRec version"
+  exit 1
+fi
+
+sudo docker build --network=host . -f docker/Dockerfile_tf212 -t  mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.12-${version}