diff --git a/Makefile b/Makefile index 5fab91cc2b..875cad68b2 100644 --- a/Makefile +++ b/Makefile @@ -178,7 +178,7 @@ mesos-watch:pre kube-sche:pre mkdir -p ${PACKAGEPATH}/bcs-k8s-master cp -R ./install/conf/bcs-k8s-master/bcs-k8s-custom-scheduler ${PACKAGEPATH}/bcs-k8s-master - go build ${LDFLAG} -o ${PACKAGEPATH}/bcs-k8s-master/bcs-k8s-custom-scheduler/bcs-k8s-custom-scheduler ./bcs-k8s/bcs-k8s-custom-scheduler/main.go + cd ./bcs-k8s/bcs-k8s-custom-scheduler && go build ${LDFLAG} -o ${WORKSPACE}/${PACKAGEPATH}/bcs-k8s-master/bcs-k8s-custom-scheduler/bcs-k8s-custom-scheduler ./main.go csi-cbs:pre mkdir -p ${PACKAGEPATH}/bcs-k8s-master diff --git a/bcs-services/bcs-service-prometheus/controller/controller.go b/bcs-services/bcs-service-prometheus/controller/controller.go index 40ce8a7da1..ce161de798 100644 --- a/bcs-services/bcs-service-prometheus/controller/controller.go +++ b/bcs-services/bcs-service-prometheus/controller/controller.go @@ -15,12 +15,13 @@ package controller import ( "encoding/json" + "os" + "strings" + "github.com/Tencent/bk-bcs/bcs-common/common/blog" commtypes "github.com/Tencent/bk-bcs/bcs-common/common/types" "github.com/Tencent/bk-bcs/bcs-services/bcs-service-prometheus/config" "github.com/Tencent/bk-bcs/bcs-services/bcs-service-prometheus/discovery" - "os" - "strings" ) const ( @@ -158,6 +159,11 @@ func (prom *PrometheusController) handleDiscoveryEvent(dInfo discovery.Discovery sdConfig, err := disc.GetPrometheusSdConfig(dInfo.Key) if err != nil { + //if serviceMonitor Not Found, then delete the specify config file + if strings.Contains(err.Error(), "Not Found") { + prom.deletePrometheusConfigFile(dInfo) + return + } blog.Errorf("discovery %s get prometheus service discovery config error %s", dInfo.Key, err.Error()) return } @@ -183,3 +189,18 @@ func (prom *PrometheusController) handleDiscoveryEvent(dInfo discovery.Discovery blog.Infof("discovery %s write config file %s success", dInfo.Key, disc.GetPromSdConfigFile(dInfo.Key)) } + +func (prom *PrometheusController) deletePrometheusConfigFile(dInfo discovery.Info) { + disc, ok := prom.discoverys[dInfo.Module] + if !ok { + blog.Errorf("not found discovery %s", dInfo.Module) + return + } + cFile := disc.GetPromSdConfigFile(dInfo.Key) + err := os.Remove(cFile) + if err != nil { + blog.Errorf("remove config file(%s) error %s", cFile, err.Error()) + return + } + blog.Infof("remove config file(%s) success", cFile) +} diff --git a/bcs-services/bcs-service-prometheus/discovery/service-monitor.go b/bcs-services/bcs-service-prometheus/discovery/service-monitor.go index 2e486d4b94..8b2c50d588 100644 --- a/bcs-services/bcs-service-prometheus/discovery/service-monitor.go +++ b/bcs-services/bcs-service-prometheus/discovery/service-monitor.go @@ -16,16 +16,17 @@ package discovery import ( "encoding/json" "fmt" - apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "path" "reflect" "regexp" "strings" "sync" + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/Tencent/bk-bcs/bcs-common/common/blog" commtypes "github.com/Tencent/bk-bcs/bcs-common/common/types" apisbkbcsv2 "github.com/Tencent/bk-bcs/bcs-mesos/pkg/apis/bkbcs/v2" @@ -56,8 +57,7 @@ type serviceMonitor struct { serviceMonitorInformer cache.SharedIndexInformer //apiextensions clientset extensionClientset *apiextensionsclient.Clientset - //initSuccess bool - + //local cache for combination of servicemonitor & bcsendpoint svrMonitors map[string]*serviceEndpoint } @@ -163,11 +163,7 @@ func (disc *serviceMonitor) Start() error { // Wait for all caches to sync. internalFactory.WaitForCacheSync(stopCh) blog.Infof("build monitorClientset for config %s success", disc.kubeconfig) - /*err = disc.initServiceMonitor() - if err!=nil { - return err - }*/ - //disc.initSuccess = true + //add k8s resources event handler functions disc.serviceMonitorInformer.AddEventHandler( cache.ResourceEventHandlerFuncs{ @@ -277,6 +273,8 @@ func (disc *serviceMonitor) OnServiceMonitorUpdate(old, cur interface{}) { disc.handlerServiceMonitorChanged(serviceM) } +// handlerServiceMonitorChanged recreate relationship between ServiceMonitor & BcsEndpoint +// no matter AddEvent or UpdateEvent func (disc *serviceMonitor) handlerServiceMonitorChanged(serviceM *apismonitorv1.ServiceMonitor) { if !disc.validateServiceMonitor(serviceM) { return @@ -306,7 +304,7 @@ func (disc *serviceMonitor) handlerServiceMonitorChanged(serviceM *apismonitorv1 } for _, v := range endpoints { if !serviceM.Match(v.Labels) { - blog.Infof("ServiceMonitor(%s) don't match BcsEndpoint(%s), and continue", serviceM.GetUuid(), v.GetUuid()) + blog.V(5).Infof("ServiceMonitor(%s) don't match BcsEndpoint(%s), and continue", serviceM.GetUuid(), v.GetUuid()) continue } o.endpoints[v.GetUuid()] = v @@ -315,7 +313,7 @@ func (disc *serviceMonitor) handlerServiceMonitorChanged(serviceM *apismonitorv1 disc.Lock() disc.svrMonitors[serviceM.GetUuid()] = o disc.Unlock() - blog.Infof("handle Update event ServiceMonitor(%s) success", serviceM.GetUuid()) + blog.Infof("handle recreate ServiceMonitor(%s) success", serviceM.GetUuid()) go disc.eventHandler(DiscoveryInfo{Module: disc.module, Key: serviceM.GetUuid()}) } @@ -342,15 +340,15 @@ func (disc *serviceMonitor) OnEndpointsAdd(obj interface{}) { } by, _ := json.Marshal(endpoint) blog.Infof("recieve BcsEndpoint(%s) Data(%s) Add event", endpoint.GetUuid(), string(by)) + disc.Lock() + defer disc.Unlock() for _, sm := range disc.svrMonitors { serviceM := sm.serviceM - if !serviceM.Match(endpoint.Labels) { - blog.V(3).Infof("ServiceMonitor(%s) don't match BcsEndpoint(%s), and continue", serviceM.GetUuid(), endpoint.GetUuid()) + if serviceM.Namespace != endpoint.Namespace || !serviceM.Match(endpoint.Labels) { + blog.V(5).Infof("ServiceMonitor(%s) don't match BcsEndpoint(%s), and continue", serviceM.GetUuid(), endpoint.GetUuid()) continue } - disc.Lock() sm.endpoints[endpoint.GetUuid()] = endpoint - disc.Unlock() blog.Infof("ServiceMonitor(%s) add selected BcsEndpoint(%s) success", serviceM.GetUuid(), endpoint.GetUuid()) // call event handler go disc.eventHandler(DiscoveryInfo{Module: disc.module, Key: serviceM.GetUuid()}) @@ -371,21 +369,20 @@ func (disc *serviceMonitor) OnEndpointsUpdate(old, cur interface{}) { } changed := checkEndpointsChanged(oldEndpoint.Spec.BcsEndpoint, curEndpoint.Spec.BcsEndpoint) if !changed { - blog.V(3).Infof("OnEndpointsUpdate BcsEndpoint(%s) don't change", oldEndpoint.GetUuid()) + blog.Infof("OnEndpointsUpdate BcsEndpoint(%s) don't change", oldEndpoint.GetUuid()) return } by, _ := json.Marshal(curEndpoint) blog.Infof("recieve BcsEndpoint(%s) Data(%s) Update event", curEndpoint.GetUuid(), string(by)) - + disc.Lock() + defer disc.Unlock() for _, sm := range disc.svrMonitors { serviceM := sm.serviceM - if !serviceM.Match(curEndpoint.Labels) { - blog.V(3).Infof("ServiceMonitor(%s) don't match BcsEndpoint(%s), and continue", serviceM.GetUuid(), curEndpoint.GetUuid()) + if serviceM.Namespace != curEndpoint.Namespace || !serviceM.Match(curEndpoint.Labels) { + blog.V(5).Infof("ServiceMonitor(%s) don't match BcsEndpoint(%s), and continue", serviceM.GetUuid(), curEndpoint.GetUuid()) continue } - disc.Lock() sm.endpoints[curEndpoint.GetUuid()] = curEndpoint - disc.Unlock() blog.Infof("ServiceMonitor(%s) update selected BcsEndpoint(%s) success", serviceM.GetUuid(), curEndpoint.GetUuid()) // call event handler go disc.eventHandler(DiscoveryInfo{Module: disc.module, Key: serviceM.GetUuid()}) @@ -420,15 +417,15 @@ func (disc *serviceMonitor) OnEndpointsDelete(obj interface{}) { return } blog.Infof("recieve BcsEndpoint(%s) Delete event", endpoint.GetUuid()) + disc.Lock() + defer disc.Unlock() for _, sm := range disc.svrMonitors { serviceM := sm.serviceM - if !serviceM.Match(endpoint.Labels) { - blog.V(3).Infof("ServiceMonitor(%s) don't match BcsEndpoint(%s), and continue", serviceM.GetUuid(), endpoint.GetUuid()) + if serviceM.Namespace != endpoint.Namespace || !serviceM.Match(endpoint.Labels) { + blog.V(5).Infof("ServiceMonitor(%s) don't match BcsEndpoint(%s), and continue", serviceM.GetUuid(), endpoint.GetUuid()) continue } - disc.Lock() delete(sm.endpoints, endpoint.GetUuid()) - disc.Unlock() blog.Infof("ServiceMonitor(%s) delete selected BcsEndpoint(%s) success", serviceM.GetUuid(), endpoint.GetUuid()) // call event handler go disc.eventHandler(DiscoveryInfo{Module: disc.module, Key: serviceM.GetUuid()})