diff --git a/Makefile b/Makefile index 0d768e2aad..9b7821b439 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ bcs-k8s: bcs-component bcs-network bcs-component:k8s-driver gamestatefulset gamedeployment hook-operator \ cc-agent csi-cbs kube-sche federated-apiserver apiserver-proxy \ apiserver-proxy-tools logbeat-sidecar webhook-server clusternet-controller mcs-agent \ - general-pod-autoscaler + general-pod-autoscaler cluster-autoscaler bcs-network:network networkpolicy ingress-controller cloud-netservice cloud-netcontroller cloud-netagent @@ -333,6 +333,11 @@ general-pod-autoscaler:pre cp -R ${BCS_CONF_COMPONENT_PATH}/bcs-general-pod-autoscaler ${PACKAGEPATH}/bcs-runtime/bcs-k8s/bcs-component cd ${BCS_COMPONENT_PATH}/bcs-general-pod-autoscaler && go mod tidy && go build ${LDFLAG} -o ${WORKSPACE}/${PACKAGEPATH}/bcs-runtime/bcs-k8s/bcs-component/bcs-general-pod-autoscaler/bcs-general-pod-autoscaler ./cmd/gpa/main.go +cluster-autoscaler:pre + mkdir -p ${PACKAGEPATH}/bcs-runtime/bcs-k8s/bcs-component + cp -R ${BCS_CONF_COMPONENT_PATH}/bcs-cluster-autoscaler ${PACKAGEPATH}/bcs-runtime/bcs-k8s/bcs-component + cd ${BCS_COMPONENT_PATH}/bcs-cluster-autoscaler && go mod tidy && go build ${LDFLAG} -o ${WORKSPACE}/${PACKAGEPATH}/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/bcs-cluster-autoscaler ./main.go + # network plugins section networkpolicy:pre cd ${BCS_NETWORK_PATH} && go mod tidy && make networkpolicy diff --git a/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/Dockerfile b/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/Dockerfile index 6667d030e6..3249eb98b6 100644 --- a/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/Dockerfile +++ b/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/Dockerfile @@ -13,7 +13,7 @@ # limitations under the License. ARG BASEIMAGE=centos:centos7 FROM $BASEIMAGE -LABEL maintainer="cmxiao " + COPY bcs-cluster-autoscaler / CMD ["/bcs-cluster-autoscaler"] diff --git a/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/cache.go b/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/cache.go index 14bec23d71..0bf01a8276 100644 --- a/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/cache.go +++ b/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/cache.go @@ -145,7 +145,8 @@ func (m *NodeGroupCache) regenerateCacheForInternal() error { if instance.NodeGroupID != groupID { continue } - ref := InstanceRef{Name: instance.NodeID} + // ref := InstanceRef{Name: instance.NodeID} + ref := InstanceRef{IP: instance.InnerIP} newCache[ref] = group newTypeCache[ref] = CreationType(CreationTypeAuto) } diff --git a/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/cloud_provider.go b/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/cloud_provider.go index f98501c5d3..8c6fea89dd 100644 --- a/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/cloud_provider.go +++ b/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/cloud_provider.go @@ -137,7 +137,8 @@ func (cloud *Provider) NodeGroups() []cloudprovider.NodeGroup { // NodeGroupForNode returns the node group for the given node. func (cloud *Provider) NodeGroupForNode(node *apiv1.Node) (cloudprovider.NodeGroup, error) { - ref, err := InstanceRefFromProviderID(node.Spec.ProviderID) + // ref, err := InstanceRefFromProviderID(node.Spec.ProviderID) + ref, err := InstanceRefFromInnerIP(node.Status.Addresses) if err != nil { return nil, err } diff --git a/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/clustermanager/clustermanager.pb.go b/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/clustermanager/clustermanager.pb.go index 8c6c7b485a..9f6c3f68a9 100644 --- a/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/clustermanager/clustermanager.pb.go +++ b/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/clustermanager/clustermanager.pb.go @@ -10,7 +10,7 @@ type NodeGroup struct { AutoScaling *AutoScalingGroup `protobuf:"bytes,6,opt,name=autoScaling,proto3" json:"autoScaling,omitempty"` LaunchTemplate *LaunchConfiguration `protobuf:"bytes,7,opt,name=launchTemplate,proto3" json:"launchTemplate,omitempty"` Labels map[string]string `protobuf:"bytes,8,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - Taints map[string]string `protobuf:"bytes,9,rep,name=taints,proto3" json:"taints,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Taints []*Taint `protobuf:"bytes,9,rep,name=taints,proto3" json:"taints,omitempty"` NodeOS string `protobuf:"bytes,10,opt,name=nodeOS,proto3" json:"nodeOS,omitempty"` Creator string `protobuf:"bytes,11,opt,name=creator,proto3" json:"creator,omitempty"` Updater string `protobuf:"bytes,12,opt,name=updater,proto3" json:"updater,omitempty"` @@ -80,7 +80,7 @@ type DataDisk struct { // InternetAccessible 公网带宽设置 type InternetAccessible struct { InternetChargeType string `protobuf:"bytes,1,opt,name=internetChargeType,proto3" json:"internetChargeType,omitempty"` - InternetMaxBandwidth string `protobuf:"bytes,2,opt,name=internetMaxBandwidth,proto3" json:"internetMaxBandwidth,omitempty"` + InternetMaxBandwidth uint32 `protobuf:"bytes,2,opt,name=internetMaxBandwidth,proto3" json:"internetMaxBandwidth,omitempty"` PublicIPAssigned bool `protobuf:"varint,3,opt,name=publicIPAssigned,proto3" json:"publicIPAssigned,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-" bson:"-"` XXX_unrecognized []byte `json:"-" bson:"-"` @@ -266,3 +266,10 @@ type TimeRange struct { XXX_unrecognized []byte `json:"-" bson:"-"` XXX_sizecache int32 `json:"-" bson:"-"` } + +// Taint 污点 +type Taint struct { + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + Effect string `protobuf:"bytes,3,opt,name=effect,proto3" json:"effect,omitempty"` +} diff --git a/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/clustermanager/common.go b/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/clustermanager/common.go index 0de447ea17..409ba795ea 100644 --- a/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/clustermanager/common.go +++ b/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/clustermanager/common.go @@ -16,6 +16,7 @@ package clustermanager import ( "bytes" "context" + "crypto/tls" "fmt" "io" "io/ioutil" @@ -36,9 +37,12 @@ type Client struct { // WithoutTLSClient init a non-tls client func WithoutTLSClient(header http.Header, url string) *Client { + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } c := &Client{ header: nil, - HttpClient: &http.Client{}, + HttpClient: &http.Client{Transport: tr}, } c.baseURL = url c.header = header diff --git a/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/node_group.go b/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/node_group.go index a7939bd03b..583e06ceb1 100644 --- a/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/node_group.go +++ b/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/node_group.go @@ -159,7 +159,8 @@ func (group *NodeGroup) DecreaseTargetSize(delta int) error { func (group *NodeGroup) Belongs(node *apiv1.Node) (bool, error) { ip := getIP(node) if len(ip) == 0 { - qcloudref, err := InstanceRefFromProviderID(node.Spec.ProviderID) + // qcloudref, err := InstanceRefFromProviderID(node.Spec.ProviderID) + qcloudref, err := InstanceRefFromInnerIP(node.Status.Addresses) if err != nil { return false, err } diff --git a/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/util.go b/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/util.go index 2d0ec652ec..c6cbaa6504 100644 --- a/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/util.go +++ b/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/cloudprovider/bcs/util.go @@ -174,6 +174,16 @@ func InstanceRefFromProviderID(id string) (*InstanceRef, error) { }, nil } +// InstanceRefFromInnerIP creates InstanceConfig object from internal address +func InstanceRefFromInnerIP(addrs []apiv1.NodeAddress) (*InstanceRef, error) { + for _, addr := range addrs { + if addr.Type == apiv1.NodeInternalIP { + return &InstanceRef{IP: addr.Address}, nil + } + } + return nil, fmt.Errorf("Cannot find internal address for node") +} + // buildNodeGroupFromSpec builds node group with value format: :: func buildNodeGroupFromSpec(value string, client clustermanager.NodePoolClientInterface) (*NodeGroup, error) { spec, err := dynamic.SpecFromString(value, true) diff --git a/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/main.go b/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/main.go index eed5cdb921..9b63e436f0 100644 --- a/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/main.go +++ b/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/main.go @@ -387,6 +387,9 @@ func run(healthCheck *metrics.HealthCheck) { healthCheck.UpdateLastActivity(loopStart) err := autoscaler.RunOnce(loopStart) + if err != nil { + klog.Warningf("scaler error: %s", err.Error()) + } if err != nil && err.Type() != errors.TransientError { metrics.RegisterError(err) } else { diff --git a/install/conf/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/Dockerfile b/install/conf/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/Dockerfile new file mode 100644 index 0000000000..072c872958 --- /dev/null +++ b/install/conf/bcs-runtime/bcs-k8s/bcs-component/bcs-cluster-autoscaler/Dockerfile @@ -0,0 +1,18 @@ +# Copyright 2016 The Kubernetes Authors. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +ARG BASEIMAGE=centos:centos7 +FROM $BASEIMAGE + +COPY bcs-cluster-autoscaler / +CMD ["/bcs-cluster-autoscaler"]