fix: service-prometheus namespace error, issue #647

(cherry picked from commit bde032e)

Makefile

@@ -178,7 +178,7 @@ mesos-watch:pre
 kube-sche:pre
 	mkdir -p ${PACKAGEPATH}/bcs-k8s-master
 	cp -R ./install/conf/bcs-k8s-master/bcs-k8s-custom-scheduler ${PACKAGEPATH}/bcs-k8s-master
-	go build ${LDFLAG} -o ${PACKAGEPATH}/bcs-k8s-master/bcs-k8s-custom-scheduler/bcs-k8s-custom-scheduler ./bcs-k8s/bcs-k8s-custom-scheduler/main.go
+	cd ./bcs-k8s/bcs-k8s-custom-scheduler && go build ${LDFLAG} -o ${WORKSPACE}/${PACKAGEPATH}/bcs-k8s-master/bcs-k8s-custom-scheduler/bcs-k8s-custom-scheduler ./main.go
 
 csi-cbs:pre
 	mkdir -p ${PACKAGEPATH}/bcs-k8s-master

bcs-services/bcs-service-prometheus/controller/controller.go

@@ -15,12 +15,13 @@ package controller
 
 import (
 	"encoding/json"
+	"os"
+	"strings"
+
 	"github.com/Tencent/bk-bcs/bcs-common/common/blog"
 	commtypes "github.com/Tencent/bk-bcs/bcs-common/common/types"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-service-prometheus/config"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-service-prometheus/discovery"
-	"os"
-	"strings"
 )
 
 const (
@@ -158,6 +159,11 @@ func (prom *PrometheusController) handleDiscoveryEvent(dInfo discovery.Discovery
 
 	sdConfig, err := disc.GetPrometheusSdConfig(dInfo.Key)
 	if err != nil {
+		//if serviceMonitor Not Found, then delete the specify config file
+		if strings.Contains(err.Error(), "Not Found") {
+			prom.deletePrometheusConfigFile(dInfo)
+			return
+		}
 		blog.Errorf("discovery %s get prometheus service discovery config error %s", dInfo.Key, err.Error())
 		return
 	}
@@ -183,3 +189,18 @@ func (prom *PrometheusController) handleDiscoveryEvent(dInfo discovery.Discovery
 
 	blog.Infof("discovery %s write config file %s success", dInfo.Key, disc.GetPromSdConfigFile(dInfo.Key))
 }
+
+func (prom *PrometheusController) deletePrometheusConfigFile(dInfo discovery.Info) {
+	disc, ok := prom.discoverys[dInfo.Module]
+	if !ok {
+		blog.Errorf("not found discovery %s", dInfo.Module)
+		return
+	}
+	cFile := disc.GetPromSdConfigFile(dInfo.Key)
+	err := os.Remove(cFile)
+	if err != nil {
+		blog.Errorf("remove config file(%s) error %s", cFile, err.Error())
+		return
+	}
+	blog.Infof("remove config file(%s) success", cFile)
+}

bcs-services/bcs-service-prometheus/discovery/service-monitor.go

@@ -16,16 +16,17 @@ package discovery
 import (
 	"encoding/json"
 	"fmt"
-	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
-	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"path"
 	"reflect"
 	"regexp"
 	"strings"
 	"sync"
 
+	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
 	"github.com/Tencent/bk-bcs/bcs-common/common/blog"
 	commtypes "github.com/Tencent/bk-bcs/bcs-common/common/types"
 	apisbkbcsv2 "github.com/Tencent/bk-bcs/bcs-mesos/pkg/apis/bkbcs/v2"
@@ -56,8 +57,7 @@ type serviceMonitor struct {
 	serviceMonitorInformer cache.SharedIndexInformer
 	//apiextensions clientset
 	extensionClientset *apiextensionsclient.Clientset
-	//initSuccess    bool
-
+	//local cache for combination of servicemonitor & bcsendpoint
 	svrMonitors map[string]*serviceEndpoint
 }
 
@@ -163,11 +163,7 @@ func (disc *serviceMonitor) Start() error {
 	// Wait for all caches to sync.
 	internalFactory.WaitForCacheSync(stopCh)
 	blog.Infof("build monitorClientset for config %s success", disc.kubeconfig)
-	/*err = disc.initServiceMonitor()
-	if err!=nil {
-		return err
-	}*/
-	//disc.initSuccess = true
+
 	//add k8s resources event handler functions
 	disc.serviceMonitorInformer.AddEventHandler(
 		cache.ResourceEventHandlerFuncs{
@@ -277,6 +273,8 @@ func (disc *serviceMonitor) OnServiceMonitorUpdate(old, cur interface{}) {
 	disc.handlerServiceMonitorChanged(serviceM)
 }
 
+// handlerServiceMonitorChanged recreate relationship between ServiceMonitor & BcsEndpoint
+// no matter AddEvent or UpdateEvent
 func (disc *serviceMonitor) handlerServiceMonitorChanged(serviceM *apismonitorv1.ServiceMonitor) {
 	if !disc.validateServiceMonitor(serviceM) {
 		return
@@ -306,7 +304,7 @@ func (disc *serviceMonitor) handlerServiceMonitorChanged(serviceM *apismonitorv1
 	}
 	for _, v := range endpoints {
 		if !serviceM.Match(v.Labels) {
-			blog.Infof("ServiceMonitor(%s) don't match BcsEndpoint(%s), and continue", serviceM.GetUuid(), v.GetUuid())
+			blog.V(5).Infof("ServiceMonitor(%s) don't match BcsEndpoint(%s), and continue", serviceM.GetUuid(), v.GetUuid())
 			continue
 		}
 		o.endpoints[v.GetUuid()] = v
@@ -315,7 +313,7 @@ func (disc *serviceMonitor) handlerServiceMonitorChanged(serviceM *apismonitorv1
 	disc.Lock()
 	disc.svrMonitors[serviceM.GetUuid()] = o
 	disc.Unlock()
-	blog.Infof("handle Update event ServiceMonitor(%s) success", serviceM.GetUuid())
+	blog.Infof("handle recreate ServiceMonitor(%s) success", serviceM.GetUuid())
 	go disc.eventHandler(DiscoveryInfo{Module: disc.module, Key: serviceM.GetUuid()})
 }
 
@@ -342,15 +340,15 @@ func (disc *serviceMonitor) OnEndpointsAdd(obj interface{}) {
 	}
 	by, _ := json.Marshal(endpoint)
 	blog.Infof("recieve BcsEndpoint(%s) Data(%s) Add event", endpoint.GetUuid(), string(by))
+	disc.Lock()
+	defer disc.Unlock()
 	for _, sm := range disc.svrMonitors {
 		serviceM := sm.serviceM
-		if !serviceM.Match(endpoint.Labels) {
-			blog.V(3).Infof("ServiceMonitor(%s) don't match BcsEndpoint(%s), and continue", serviceM.GetUuid(), endpoint.GetUuid())
+		if serviceM.Namespace != endpoint.Namespace || !serviceM.Match(endpoint.Labels) {
+			blog.V(5).Infof("ServiceMonitor(%s) don't match BcsEndpoint(%s), and continue", serviceM.GetUuid(), endpoint.GetUuid())
 			continue
 		}
-		disc.Lock()
 		sm.endpoints[endpoint.GetUuid()] = endpoint
-		disc.Unlock()
 		blog.Infof("ServiceMonitor(%s) add selected BcsEndpoint(%s) success", serviceM.GetUuid(), endpoint.GetUuid())
 		// call event handler
 		go disc.eventHandler(DiscoveryInfo{Module: disc.module, Key: serviceM.GetUuid()})
@@ -371,21 +369,20 @@ func (disc *serviceMonitor) OnEndpointsUpdate(old, cur interface{}) {
 	}
 	changed := checkEndpointsChanged(oldEndpoint.Spec.BcsEndpoint, curEndpoint.Spec.BcsEndpoint)
 	if !changed {
-		blog.V(3).Infof("OnEndpointsUpdate BcsEndpoint(%s) don't change", oldEndpoint.GetUuid())
+		blog.Infof("OnEndpointsUpdate BcsEndpoint(%s) don't change", oldEndpoint.GetUuid())
 		return
 	}
 	by, _ := json.Marshal(curEndpoint)
 	blog.Infof("recieve BcsEndpoint(%s) Data(%s) Update event", curEndpoint.GetUuid(), string(by))
-
+	disc.Lock()
+	defer disc.Unlock()
 	for _, sm := range disc.svrMonitors {
 		serviceM := sm.serviceM
-		if !serviceM.Match(curEndpoint.Labels) {
-			blog.V(3).Infof("ServiceMonitor(%s) don't match BcsEndpoint(%s), and continue", serviceM.GetUuid(), curEndpoint.GetUuid())
+		if serviceM.Namespace != curEndpoint.Namespace || !serviceM.Match(curEndpoint.Labels) {
+			blog.V(5).Infof("ServiceMonitor(%s) don't match BcsEndpoint(%s), and continue", serviceM.GetUuid(), curEndpoint.GetUuid())
 			continue
 		}
-		disc.Lock()
 		sm.endpoints[curEndpoint.GetUuid()] = curEndpoint
-		disc.Unlock()
 		blog.Infof("ServiceMonitor(%s) update selected BcsEndpoint(%s) success", serviceM.GetUuid(), curEndpoint.GetUuid())
 		// call event handler
 		go disc.eventHandler(DiscoveryInfo{Module: disc.module, Key: serviceM.GetUuid()})
@@ -420,15 +417,15 @@ func (disc *serviceMonitor) OnEndpointsDelete(obj interface{}) {
 		return
 	}
 	blog.Infof("recieve BcsEndpoint(%s) Delete event", endpoint.GetUuid())
+	disc.Lock()
+	defer disc.Unlock()
 	for _, sm := range disc.svrMonitors {
 		serviceM := sm.serviceM
-		if !serviceM.Match(endpoint.Labels) {
-			blog.V(3).Infof("ServiceMonitor(%s) don't match BcsEndpoint(%s), and continue", serviceM.GetUuid(), endpoint.GetUuid())
+		if serviceM.Namespace != endpoint.Namespace || !serviceM.Match(endpoint.Labels) {
+			blog.V(5).Infof("ServiceMonitor(%s) don't match BcsEndpoint(%s), and continue", serviceM.GetUuid(), endpoint.GetUuid())
 			continue
 		}
-		disc.Lock()
 		delete(sm.endpoints, endpoint.GetUuid())
-		disc.Unlock()
 		blog.Infof("ServiceMonitor(%s) delete selected BcsEndpoint(%s) success", serviceM.GetUuid(), endpoint.GetUuid())
 		// call event handler
 		go disc.eventHandler(DiscoveryInfo{Module: disc.module, Key: serviceM.GetUuid()})