save temp

Signed-off-by: clyi <[email protected]>

Makefile

@@ -134,9 +134,8 @@ build-go-arm:
 	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(GO_BUILD_FLAGS) -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-controller -v ./cmd/controller
 
 .PHONY: build-kube-ovn
-build-kube-ovn: build-debug build-go
+build-kube-ovn: build-go
 	docker build -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG) --build-arg VERSION=$(RELEASE_TAG) -f dist/images/Dockerfile dist/images/
-	docker build -t $(REGISTRY)/kube-ovn:$(LEGACY_TAG) --build-arg VERSION=$(LEGACY_TAG) -f dist/images/Dockerfile dist/images/
 
 .PHONY: build-kube-ovn-dpdk
 build-kube-ovn-dpdk: build-go

dist/images/Dockerfile

@@ -1,7 +1,7 @@
 # syntax = docker/dockerfile:experimental
 ARG VERSION
 ARG BASE_TAG=$VERSION
-FROM kubeovn/kube-ovn-base:$BASE_TAG AS setcap
+FROM yichanglu/kube-ovn-base:$BASE_TAG AS setcap
 
 COPY *.sh /kube-ovn/
 COPY kubectl-ko /kube-ovn/kubectl-ko
@@ -22,7 +22,7 @@ RUN ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-monitor && \
     setcap CAP_NET_RAW,CAP_NET_BIND_SERVICE+eip /kube-ovn/kube-ovn-controller && \
     setcap CAP_NET_ADMIN,CAP_NET_RAW,CAP_NET_BIND_SERVICE,CAP_SYS_ADMIN+eip /kube-ovn/kube-ovn-daemon
 
-FROM kubeovn/kube-ovn-base:$BASE_TAG
+FROM yichanglu/kube-ovn-base:$BASE_TAG
 
 COPY --chmod=0644 logrotate/* /etc/logrotate.d/
 COPY grace_stop_ovn_controller /usr/share/ovn/scripts/grace_stop_ovn_controller

dist/images/install.sh

@@ -48,12 +48,14 @@ OVSDB_INACTIVITY_TIMEOUT=${OVSDB_INACTIVITY_TIMEOUT:-10}
 ENABLE_LIVE_MIGRATION_OPTIMIZE=${ENABLE_LIVE_MIGRATION_OPTIMIZE:-true}
 
 # debug
-DEBUG_WRAPPER=${DEBUG_WRAPPER:-}
+DEBUG_WRAPPER=${DEBUG_WRAPPER:-true}
 RUN_AS_USER=65534 # run as nobody
 if [ "$ENABLE_OVN_IPSEC" = "true" -o -n "$DEBUG_WRAPPER" ]; then
   RUN_AS_USER=0
 fi
 
+RUN_AS_USER=0
+
 KUBELET_DIR=${KUBELET_DIR:-/var/lib/kubelet}
 LOG_DIR=${LOG_DIR:-/var/log}
 
@@ -2810,6 +2812,8 @@ spec:
                   type: boolean
                 enableMulticastSnoop:
                   type: boolean
+                isExternalLBAddressPool:
+                  type: boolean
                 routeTable:
                   type: string
                 namespaceSelectors:
@@ -3666,6 +3670,7 @@ rules:
       - ovn-eips/status
       - nodes
       - pods
+      - vips
     verbs:
       - get
       - list
@@ -3957,7 +3962,7 @@ spec:
           - /kube-ovn/start-db.sh
           securityContext:
             runAsUser: ${RUN_AS_USER}
-            privileged: false
+            privileged: true
             capabilities:
               add:
                 - NET_BIND_SERVICE
@@ -4303,7 +4308,7 @@ spec:
           - /kube-ovn/start-ovs.sh
           securityContext:
             runAsUser: ${RUN_AS_USER}
-            privileged: false
+            privileged: true
             capabilities:
               add:
                 - NET_ADMIN
@@ -4731,7 +4736,7 @@ spec:
           - --image=$REGISTRY/kube-ovn:$VERSION
           securityContext:
             runAsUser: ${RUN_AS_USER}
-            privileged: false
+            privileged: true
             capabilities:
               add:
                 - NET_BIND_SERVICE
@@ -4924,7 +4929,7 @@ spec:
           - --set-vxlan-tx-off=$SET_VXLAN_TX_OFF
         securityContext:
           runAsUser: 0
-          privileged: false
+          privileged: true
           capabilities:
             add:
               - NET_ADMIN
@@ -5139,7 +5144,7 @@ spec:
           imagePullPolicy: $IMAGE_PULL_POLICY
           securityContext:
             runAsUser: ${RUN_AS_USER}
-            privileged: false
+            privileged: true
             capabilities:
               add:
                 - NET_BIND_SERVICE
@@ -5290,7 +5295,7 @@ spec:
           - --enable-metrics=$ENABLE_METRICS
           securityContext:
             runAsUser: ${RUN_AS_USER}
-            privileged: false
+            privileged: true
             capabilities:
               add:
                 - NET_BIND_SERVICE
@@ -5513,7 +5518,7 @@ spec:
           - --alsologtostderr=true
           securityContext:
             runAsUser: ${RUN_AS_USER}
-            privileged: false
+            privileged: true
             capabilities:
               add:
                 - NET_BIND_SERVICE

pkg/apis/kubeovn/v1/subnet.go

@@ -80,11 +80,12 @@ type SubnetSpec struct {
 
 	NatOutgoingPolicyRules []NatOutgoingPolicyRule `json:"natOutgoingPolicyRules,omitempty"`
 
-	U2OInterconnectionIP string `json:"u2oInterconnectionIP,omitempty"`
-	U2OInterconnection   bool   `json:"u2oInterconnection,omitempty"`
-	EnableLb             *bool  `json:"enableLb,omitempty"`
-	EnableEcmp           bool   `json:"enableEcmp,omitempty"`
-	EnableMulticastSnoop bool   `json:"enableMulticastSnoop,omitempty"`
+	U2OInterconnectionIP    string `json:"u2oInterconnectionIP,omitempty"`
+	U2OInterconnection      bool   `json:"u2oInterconnection,omitempty"`
+	EnableLb                *bool  `json:"enableLb,omitempty"`
+	EnableEcmp              bool   `json:"enableEcmp,omitempty"`
+	EnableMulticastSnoop    bool   `json:"enableMulticastSnoop,omitempty"`
+	IsExternalLBAddressPool bool   `json:"isExternalLBAddressPool,omitempty"`
 
 	RouteTable         string                 `json:"routeTable,omitempty"`
 	NamespaceSelectors []metav1.LabelSelector `json:"namespaceSelectors,omitempty"`

pkg/controller/config.go

@@ -93,6 +93,7 @@ type Configuration struct {
 	EnableEcmp                  bool
 	EnableKeepVMIP              bool
 	EnableLbSvc                 bool
+	EnableLbSvcPolicyLocal      bool
 	EnableMetrics               bool
 	EnableANP                   bool
 	EnableOVNIPSec              bool
@@ -175,6 +176,7 @@ func ParseFlags() (*Configuration, error) {
 		argEnableEcmp                  = pflag.Bool("enable-ecmp", false, "Enable ecmp route for centralized subnet")
 		argKeepVMIP                    = pflag.Bool("keep-vm-ip", true, "Whether to keep ip for kubevirt pod when pod is rebuild")
 		argEnableLbSvc                 = pflag.Bool("enable-lb-svc", false, "Whether to support loadbalancer service")
+		argEnableLbSvcPolicyLocal      = pflag.Bool("enable-lb-svc-policy-local", true, "Whether to support external loadbalancer")
 		argEnableMetrics               = pflag.Bool("enable-metrics", true, "Whether to support metrics query")
 		argEnableANP                   = pflag.Bool("enable-anp", false, "Enable support for admin network policy and baseline admin network policy")
 		argEnableOVNIPSec              = pflag.Bool("enable-ovn-ipsec", false, "Whether to enable ovn ipsec")
@@ -271,6 +273,7 @@ func ParseFlags() (*Configuration, error) {
 		GCInterval:                     *argGCInterval,
 		InspectInterval:                *argInspectInterval,
 		EnableLbSvc:                    *argEnableLbSvc,
+		EnableLbSvcPolicyLocal:         *argEnableLbSvcPolicyLocal,
 		EnableMetrics:                  *argEnableMetrics,
 		EnableOVNIPSec:                 *argEnableOVNIPSec,
 		EnableLiveMigrationOptimize:    *argEnableLiveMigrationOptimize,

pkg/controller/endpoint.go

@@ -75,6 +75,7 @@ func (c *Controller) handleUpdateEndpoint(key string) error {
 		vip, vpcName, subnetName string
 		ok                       bool
 		ignoreHealthCheck        = true
+		isPreferLocalBackend     = false
 	)
 
 	if vip, ok = svc.Annotations[util.SwitchLBRuleVipsAnnotation]; ok {
@@ -93,6 +94,20 @@ func (c *Controller) handleUpdateEndpoint(key string) error {
 		return nil
 	}
 
+	// 注意这些东西只有在ovn lb开关打开的时候才能用
+	if svc.Spec.Type == v1.ServiceTypeLoadBalancer && svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
+		if externalIP := util.GetLoadBalancerIP(*svc); err == nil && externalIP != "" {
+			lbVips = append(lbVips, externalIP)
+		} else if err != nil {
+			klog.Errorf("failed to get external load balancer IP for service %s/%s: %v", namespace, name, err)
+			return err
+		}
+		isPreferLocalBackend = true
+	} else if svc.Spec.Type == v1.ServiceTypeClusterIP && svc.Spec.InternalTrafficPolicy != nil && *svc.Spec.InternalTrafficPolicy == v1.ServiceInternalTrafficPolicyLocal {
+		isPreferLocalBackend = true
+	}
+
+	klog.Errorf("clyi lbVips %v", lbVips)
 	if pods, err = c.podsLister.Pods(namespace).List(labels.Set(svc.Spec.Selector).AsSelector()); err != nil {
 		klog.Errorf("failed to get pods for service %s in namespace %s: %v", name, namespace, err)
 		return err
@@ -157,19 +172,40 @@ func (c *Controller) handleUpdateEndpoint(key string) error {
 				backends                 []string
 				ipPortMapping, externals map[string]string
 			)
-
-			if !ignoreHealthCheck {
+			isGenIPPortMapping := !ignoreHealthCheck || isPreferLocalBackend
+			if isGenIPPortMapping {
 				if checkIP, err = c.getHealthCheckVip(subnetName, lbVip); err != nil {
 					klog.Error(err)
 					return err
 				}
+
+				// subnet, err := c.subnetsLister.Get(subnetName)
+				// if err != nil {
+				// 	klog.Errorf("failed to get subnet %s: %v", subnetName, err)
+				// 	return err
+				// }
+
+				// if subnet.Spec.IsExternalLBAddressPool {
+				// 	vipName := fmt.Sprintf("%s.%s", svc.Name, svc.Namespace)
+				// 	vip := &kubeovnv1.Vip{
+				// 		ObjectMeta: metav1.ObjectMeta{
+				// 			Name: vipName,
+				// 		},
+				// 		Spec: kubeovnv1.VipSpec{
+				// 			Subnet: subnetName,
+				// 		},
+				// 	}
+				// 	if _, err = c.config.KubeOvnClient.KubeovnV1().Vips().Create(context.Background(), vip, metav1.CreateOptions{}); err != nil {
+				// 		klog.Errorf("failed to create vip %s, %v", vipName, err)
+				// 		return err
+				// 	}
+				// }
+
 				externals = map[string]string{
 					util.SwitchLBRuleSubnet: subnetName,
 				}
 			}
-
-			ipPortMapping, backends = getIPPortMappingBackend(ep, pods, port, lbVip, checkIP, ignoreHealthCheck)
-
+			ipPortMapping, backends = getIPPortMappingBackend(ep, pods, port, lbVip, checkIP, isGenIPPortMapping)
 			// for performance reason delete lb with no backends
 			if len(backends) != 0 {
 				vip = util.JoinHostPort(lbVip, port.Port)
@@ -178,6 +214,14 @@ func (c *Controller) handleUpdateEndpoint(key string) error {
 					klog.Errorf("failed to add vip %s with backends %s to LB %s: %v", lbVip, backends, lb, err)
 					return err
 				}
+
+				if isPreferLocalBackend {
+					if err = c.OVNNbClient.LoadBalancerUpdateIPPortMapping(lb, vip, ipPortMapping); err != nil {
+						klog.Errorf("failed to update ip port mapping %s for vip %s to LB %s: %v", ipPortMapping, vip, lb, err)
+						return err
+					}
+				}
+
 				if !ignoreHealthCheck && len(ipPortMapping) != 0 {
 					klog.Infof("add health check ip port mapping %v to LB %s", ipPortMapping, lb)
 					if err = c.OVNNbClient.LoadBalancerAddHealthCheck(lb, vip, ignoreHealthCheck, ipPortMapping, externals); err != nil {
@@ -321,7 +365,7 @@ func (c *Controller) getHealthCheckVip(subnetName, lbVip string) (string, error)
 	return checkIP, nil
 }
 
-func getIPPortMappingBackend(endpoints *v1.Endpoints, pods []*v1.Pod, servicePort v1.ServicePort, serviceIP, checkVip string, ignoreHealthCheck bool) (map[string]string, []string) {
+func getIPPortMappingBackend(endpoints *v1.Endpoints, pods []*v1.Pod, servicePort v1.ServicePort, serviceIP, checkVip string, isGenIPPortMapping bool) (map[string]string, []string) {
 	var (
 		ipPortMapping = map[string]string{}
 		backends      = []string{}
@@ -341,7 +385,7 @@ func getIPPortMappingBackend(endpoints *v1.Endpoints, pods []*v1.Pod, servicePor
 		}
 
 		for _, address := range subset.Addresses {
-			if !ignoreHealthCheck && address.TargetRef.Name != "" {
+			if isGenIPPortMapping && address.TargetRef.Name != "" {
 				ipName := fmt.Sprintf("%s.%s", address.TargetRef.Name, endpoints.Namespace)
 				ipPortMapping[address.IP] = fmt.Sprintf(util.HealthCheckNamedVipTemplate, ipName, checkVip)
 			}

pkg/controller/init.go

@@ -251,6 +251,12 @@ func (c *Controller) initLB(name, protocol string, sessionAffinity bool) error {
 		}
 	}
 
+	err = c.OVNNbClient.SetLoadBalancerPreferLocalBackend(name, c.config.EnableLbSvcPolicyLocal)
+	if err != nil {
+		klog.Errorf("failed to set prefer local backend for load balancer %s: %v", name, err)
+		return err
+	}
+
 	return nil
 }
 

pkg/controller/subnet.go

@@ -681,6 +681,11 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error {
 
 	needRouter := subnet.Spec.Vlan == "" || subnet.Spec.LogicalGateway ||
 		(subnet.Status.U2OInterconnectionIP != "" && subnet.Spec.U2OInterconnection)
+
+	if subnet.Spec.Vlan != "" && subnet.Spec.IsExternalLBAddressPool {
+		needRouter = true
+	}
+
 	// 1. overlay subnet, should add lrp, lrp ip is subnet gw
 	// 2. underlay subnet use logical gw, should add lrp, lrp ip is subnet gw
 	randomAllocateGW := !subnet.Spec.LogicalGateway && vpc.Spec.EnableExternal && subnet.Name == c.config.ExternalGatewaySwitch
@@ -1852,8 +1857,12 @@ func (c *Controller) reconcileSubnetSpecialIPs(subnet *kubeovnv1.Subnet) (bool,
 	if subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway {
 		u2oInterconnName := fmt.Sprintf(util.U2OInterconnName, subnet.Spec.Vpc, subnet.Name)
 		u2oInterconnLrpName := fmt.Sprintf("%s-%s", subnet.Spec.Vpc, subnet.Name)
+
+		klog.Errorf("changlu subnet.Spec.IsExternalLBAddressPool %v ", subnet.Spec.IsExternalLBAddressPool)
+		needAllocateU2OIP := false
+		needAllocateU2OIP = subnet.Spec.U2OInterconnection || subnet.Spec.IsExternalLBAddressPool
 		var v4ip, v6ip string
-		if subnet.Spec.U2OInterconnection {
+		if needAllocateU2OIP {
 			v4ip, v6ip, _, err = c.acquireU2OIP(subnet, u2oInterconnName, u2oInterconnLrpName)
 			if err != nil {
 				return isU2OIPChanged, isMcastQuerierIPChanged, err
@@ -1862,7 +1871,7 @@ func (c *Controller) reconcileSubnetSpecialIPs(subnet *kubeovnv1.Subnet) (bool,
 			if v4ip != "" || v6ip != "" {
 				isU2OIPChanged = true
 			}
-		} else if subnet.Status.U2OInterconnectionIP != "" {
+		} else if !needAllocateU2OIP && subnet.Status.U2OInterconnectionIP != "" {
 			err = c.releaseU2OIP(subnet, u2oInterconnName)
 			if err != nil {
 				return isU2OIPChanged, isMcastQuerierIPChanged, err