From a6e9ad80bf70f0edd3cf3f207f0e243999606b9d Mon Sep 17 00:00:00 2001 From: denglouping <359150423@qq.com> Date: Fri, 1 Mar 2024 16:39:52 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E9=9B=86=E7=BE=A4apiserver=E8=AF=81?= =?UTF-8?q?=E4=B9=A6=E6=94=AF=E6=8C=81=E7=9B=B4=E6=8E=A5=E9=85=8D=E7=BD=AE?= =?UTF-8?q?IP=E5=92=8C=E5=9F=9F=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bcs-ops/functions/k8s.sh | 47 ++++++++++++++++++++++++++++++++ bcs-ops/k8s/render_kubeadm | 4 +++ bcs-ops/readme.md | 37 +++++++++++++------------ bcs-ops/system/config_envfile.sh | 6 ++++ 4 files changed, 76 insertions(+), 18 deletions(-) diff --git a/bcs-ops/functions/k8s.sh b/bcs-ops/functions/k8s.sh index 4d2208375b..1746668ca1 100644 --- a/bcs-ops/functions/k8s.sh +++ b/bcs-ops/functions/k8s.sh @@ -59,6 +59,53 @@ k8s::safe_add_helmrepo() { # add vip success - return 0 # add vip fail - return 1 ####################################### +k8s::config_master_vip() { + op_type=$1 + excute=$2 + vips=$3 + path=$4 + + if [[ "${op_type}" == "list" ]];then + path=$2 + fi + + if [[ -z "${path}" ]];then + path=$(kubeadm_config_file="/tmp/kubeadm-$(date +%Y-%m-%d).yaml") + kubectl -n kube-system get configmap kubeadm-config -o jsonpath='{.data.ClusterConfiguration}' >"${path}" + fi + + case "$op_type" in + add) + for vip in ${vips//,/ };do + if [[ -n "${vip}" ]];then + yq e -i '(select(.apiServer != null)|.apiServer.certSANs) += ["'${vip}'"]' ${path} + fi + done + ;; + delete) + for vip in ${vips//,/ };do + if [[ -n "${vip}" ]];then + yq e 'del(.apiServer.certSANs[] | select(. == "'${vip}'"))' -i ${path} + fi + done + ;; + list) + yq 'select(.apiServer.certSANs != null)|.apiServer.certSANs' ${path} + ;; + *) + export ERR_CODE=1 + utils::log "ERROR" "unkown command: $1" + ;; + esac + + if [[ "${excute}" == "true" ]] && [[ "${op_type}" != "list" ]];then + kubeadm init phase certs apiserver --config "${path}" \ + || utils::log "ERROR" "failed to ${op_type} ${vips} ${path}" + fi + + utils::log "OK" "${op_type} ${vips} ${path}" +} + k8s::add_vip_to_cert() { vip=$1 local kubeadm_config_file diff --git a/bcs-ops/k8s/render_kubeadm b/bcs-ops/k8s/render_kubeadm index e22f56b3e1..4f37ab49d5 100755 --- a/bcs-ops/k8s/render_kubeadm +++ b/bcs-ops/k8s/render_kubeadm @@ -145,6 +145,8 @@ IPv6EOF ) --- EOF + + k8s::config_master_vip add false ${EXTERNAL_VIP},${EXTERNAL_HOST} ${config_file} } render_kubelet() { @@ -316,6 +318,8 @@ safe_source() { fi } +safe_source "${ROOT_DIR}/functions/k8s.sh" + main() { local source_files ipv6_status cri_type source_files=("${ROOT_DIR}/functions/utils.sh" "${ROOT_DIR}/env/bcs.env") diff --git a/bcs-ops/readme.md b/bcs-ops/readme.md index e84d94b393..067c9038ab 100644 --- a/bcs-ops/readme.md +++ b/bcs-ops/readme.md @@ -79,7 +79,7 @@ set +x 1. 通过`set -a 命令`配置环境变量，环境变量配置见[`环境变量`](#环境变量) 2. `./bcs-ops -r bcsenv` 在第一台主机（后称中控机）上渲染配置文件 `env/bcs.env` -3. 在中控机上启动集群控制平面：`./bcs-ops --instal master`，集群启动成功后会显示加入集群的指令 +3. 在中控机上启动集群控制平面：`./bcs-ops --install master`，集群启动成功后会显示加入集群的指令 4. 集群加入指令有效期为 1 小时，中控机执行 `./bcs-ops --render joincmd` 可再次渲染生成加入集群的指令，渲染结果如下所示 ```plaintext @@ -196,23 +196,24 @@ set +x #### apiserver ha 环境变量 -| 环境变量 | 默认值 | 说明 | -| ------------------------- | --------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `ENABLE_APISERVER_HA` | `false` | apiserver ha 模式，默认关闭。 | -| `APISERVER_HA_MODE` | `bcs-apiserver-proxy` | 模式选择，支持 [bcs-apiserver-proxy](https://github.com/TencentBlueKing/bk-bcs/blob/625be3183d99ee3500123016a6dea99d78165565/docs/features/bcs-apiserver-proxy/bcs-apiserver-proxy.md#L1)`​[kube-vip](https://kube-vip.io/)` | -| `VIP` | | VIP 地址，可配置与集群内不冲突的 ip 地址 | -| `VS_PORT` | `6443` | bap 代理端口 | -| `APISERVER_PROXY_VERSION` | `v1.29.0-alpha.130-tencent` | bap 镜像版本 | -| `PROXY_TOOL_PATH` | `/usr/bin` | bap 工具安装目录 | -| `PERSIST_DIR` | `/root/.bcs` | bap 持久化目录 | -| `LVS_SCHEDULER` | `rr` | bap 负载均衡策略 | -| `MANAGER_INTERVAL` | `10` | bap 监听时间 | -| `DEBUG_MODE` | `true` | bap DEBUG 模式 默认开启 | -| `LOG_LEVEL` | `3` | bap 日志等级 | -| `KUBE_VIP_VERSION` | `v0.5.12` | kube-vip 镜像版本 | -| `BIND_INTERFACE` | `""` | kube-vip 绑定网卡名 | -| `VIP_CIDR` | `32` | VIP CIDR 掩码长度 | - +| 环境变量 | 默认值 | 说明 | +| ------------------------- | --------------------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `ENABLE_APISERVER_HA` | `false` | apiserver ha 模式，默认关闭。 | +| `APISERVER_HA_MODE` | `bcs-apiserver-proxy` | 模式选择，支持 [bcs-apiserver-proxy](https://github.com/TencentBlueKing/bk-bcs/blob/625be3183d99ee3500123016a6dea99d78165565/docs/features/bcs-apiserver-proxy/bcs-apiserver-proxy.md#L1), [kube-vip](https://kube-vip.io/) | +| `VIP` | | VIP 地址，可配置与集群内不冲突的 ip 地址 | +| `VS_PORT` | `6443` | bap 代理端口 | +| `APISERVER_PROXY_VERSION` | `v1.29.0-alpha.130-tencent` | bap 镜像版本 | +| `PROXY_TOOL_PATH` | `/usr/bin` | bap 工具安装目录 | +| `PERSIST_DIR` | `/root/.bcs` | bap 持久化目录 | +| `LVS_SCHEDULER` | `rr` | bap 负载均衡策略 | +| `MANAGER_INTERVAL` | `10` | bap 监听时间 | +| `DEBUG_MODE` | `true` | bap DEBUG 模式 默认开启 | +| `LOG_LEVEL` | `3` | bap 日志等级 | +| `KUBE_VIP_VERSION` | `v0.5.12` | kube-vip 镜像版本 | +| `BIND_INTERFACE` | `""` | kube-vip 绑定网卡名 | +| `VIP_CIDR` | `32` | VIP CIDR 掩码长度 | +| `EXTERNAL_VIP` | `""` | 外部LB ip | +| `EXTERNAL_HOST` | `""` | 外部域名解析域名 | ### 示例 #### a. 选择 `1.24.15` 的 k8s 版本 ，并使用 `containerd` 作为容器运行时 diff --git a/bcs-ops/system/config_envfile.sh b/bcs-ops/system/config_envfile.sh index 8acfb70959..139a2d3839 100755 --- a/bcs-ops/system/config_envfile.sh +++ b/bcs-ops/system/config_envfile.sh @@ -139,6 +139,9 @@ init_env() { KUBE_VIP_VERSION=${KUBE_VIP_VERSION:-"v0.5.12"} BIND_INTERFACE=${BIND_INTERFACE:-} VIP_CIDR=${VIP_CIDR:-"32"} + ## external-vip + EXTERNAL_VIP=${EXTERNAL_VIP:-} + EXTERNAL_HOST=${EXTERNAL_HOST:-} ## multus ENABLE_MULTUS_HA=${ENABLE_MULTUS_HA:-"true"} } @@ -336,6 +339,9 @@ DEBUG_MODE="${DEBUG_MODE}" KUBE_VIP_VERSION="${KUBE_VIP_VERSION}" BIND_INTERFACE="${BIND_INTERFACE}" VIP_CIDR="${VIP_CIDR}" +## external-vip +EXTERNAL_VIP="${EXTERNAL_VIP}" +EXTERNAL_HOST="${EXTERNAL_HOST}" ## multus ENABLE_MULTUS_HA="${ENABLE_MULTUS_HA}" # bcs config end From d2a2868b31877b04328e57a90298b3c5604f5ee5 Mon Sep 17 00:00:00 2001 From: denglouping <359150423@qq.com> Date: Mon, 4 Mar 2024 20:16:48 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E9=9B=86=E7=BE=A4apiserver=E8=AF=81?= =?UTF-8?q?=E4=B9=A6=E6=94=AF=E6=8C=81=E7=9B=B4=E6=8E=A5=E9=85=8D=E7=BD=AE?= =?UTF-8?q?IP=E5=92=8C=E5=9F=9F=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bcs-ops/functions/k8s.sh | 27 +++++++++++++++++---------- bcs-ops/k8s/render_kubeadm | 4 +--- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/bcs-ops/functions/k8s.sh b/bcs-ops/functions/k8s.sh index 1746668ca1..f8cb045fcf 100644 --- a/bcs-ops/functions/k8s.sh +++ b/bcs-ops/functions/k8s.sh @@ -54,20 +54,19 @@ k8s::safe_add_helmrepo() { ####################################### # add vip to K8S apiserver certs # Arguments: -# $1: vip +# $1: op_type +# $2: excute (if op_type is $1, then $2 is path) +# $3: vips +# $4: path # Return: -# add vip success - return 0 -# add vip fail - return 1 +# excute success - return 0 +# excute - return 1 ####################################### k8s::config_master_vip() { op_type=$1 excute=$2 - vips=$3 - path=$4 - - if [[ "${op_type}" == "list" ]];then - path=$2 - fi + vips=${3:-""} + path=${4:-$2} if [[ -z "${path}" ]];then path=$(kubeadm_config_file="/tmp/kubeadm-$(date +%Y-%m-%d).yaml") @@ -78,7 +77,7 @@ k8s::config_master_vip() { add) for vip in ${vips//,/ };do if [[ -n "${vip}" ]];then - yq e -i '(select(.apiServer != null)|.apiServer.certSANs) += ["'${vip}'"]' ${path} + yq e -i '(select(.apiServer != null)|.apiServer.certSANs) += ["'${vip}'"]| select(.apiServer != null)|.apiServer.certSANs|= unique' ${path} fi done ;; @@ -106,6 +105,14 @@ k8s::config_master_vip() { utils::log "OK" "${op_type} ${vips} ${path}" } +####################################### +# add vip to K8S apiserver certs +# Arguments: +# $1: vip +# Return: +# add vip success - return 0 +# add vip fail - return 1 +####################################### k8s::add_vip_to_cert() { vip=$1 local kubeadm_config_file diff --git a/bcs-ops/k8s/render_kubeadm b/bcs-ops/k8s/render_kubeadm index 4f37ab49d5..11dd8f5981 100755 --- a/bcs-ops/k8s/render_kubeadm +++ b/bcs-ops/k8s/render_kubeadm @@ -318,11 +318,9 @@ safe_source() { fi } -safe_source "${ROOT_DIR}/functions/k8s.sh" - main() { local source_files ipv6_status cri_type - source_files=("${ROOT_DIR}/functions/utils.sh" "${ROOT_DIR}/env/bcs.env") + source_files=("${ROOT_DIR}/functions/utils.sh" "${ROOT_DIR}/env/bcs.env" "${ROOT_DIR}/functions/k8s.sh") for file in "${source_files[@]}"; do safe_source "$file" done