From f857f0ffa67a25aa0a6851a4248d5acee14aefe6 Mon Sep 17 00:00:00 2001 From: abstractmj Date: Mon, 6 Jul 2020 02:38:07 +0800 Subject: [PATCH 1/2] feature: support dynamic scale up in init function, support bridge rout; issue #515 --- .../bcs-network/qcloud-eip/conf/conf.go | 22 ++-- .../bcs-network/qcloud-eip/eip/eip.go | 124 +++++++++++++----- .../bcs-network/qcloud-eip/eip/instance.go | 6 + .../bcs-network/qcloud-eip/eip/netservice.go | 11 +- .../bcs-network/qcloud-eip/eip/util.go | 37 +++++- .../bcs-network/qcloud-eip/eip/vpc.go | 6 + 6 files changed, 166 insertions(+), 40 deletions(-) diff --git a/bcs-services/bcs-network/qcloud-eip/conf/conf.go b/bcs-services/bcs-network/qcloud-eip/conf/conf.go index bc309c090e..889ef9adb7 100644 --- a/bcs-services/bcs-network/qcloud-eip/conf/conf.go +++ b/bcs-services/bcs-network/qcloud-eip/conf/conf.go @@ -35,14 +35,20 @@ type NetArgs struct { // NetConf net config type NetConf struct { types.NetConf - Master string `json:"master"` - ENIPrefix string `json:"eniPrefix"` - ClusterID string `json:"clusterId"` - Region string `json:"region"` - Secret string `json:"secret"` - UUID string `json:"uuid"` - SubnetID string `json:"subnetId,omitempty"` - MTU int `json:"mtu,omitempty"` + TencentCloudCVMDomain string `json:"tencentcloudCVMDomain"` + TencentCloudVPCDomain string `json:"tencentcloudVPCDomain"` + Master string `json:"master"` + ENIPrefix string `json:"eniPrefix"` + ClusterID string `json:"clusterId"` + Region string `json:"region"` + Secret string `json:"secret"` + UUID string `json:"uuid"` + SubnetID string `json:"subnetId,omitempty"` + MTU int `json:"mtu,omitempty"` + // Design: If both bridge network mode and elastic network card mode exist + // copy route to bridge into each eni route table, otherwise pod with eni can + // not communicate with the bridge-mode pod + BridgeName string `json:"bridgeName"` NetService *NetArgs `json:"netservice,omitempty"` Args *bcsconf.CNIArgs } diff --git a/bcs-services/bcs-network/qcloud-eip/eip/eip.go b/bcs-services/bcs-network/qcloud-eip/eip/eip.go index 5092dc8902..cc4828c66a 100644 --- a/bcs-services/bcs-network/qcloud-eip/eip/eip.go +++ b/bcs-services/bcs-network/qcloud-eip/eip/eip.go @@ -13,9 +13,6 @@ package eip import ( - "bk-bcs/bcs-common/common/blog" - netsvc "bk-bcs/bcs-services/bcs-netservice/pkg/netservice/types" - "bk-bcs/bcs-services/bcs-network/qcloud-eip/conf" "fmt" "net" "os" @@ -29,11 +26,17 @@ import ( "github.com/containernetworking/plugins/pkg/ipam" "github.com/containernetworking/plugins/pkg/ns" "github.com/vishvananda/netlink" + + "bk-bcs/bcs-common/common/blog" + netsvc "bk-bcs/bcs-services/bcs-netservice/pkg/netservice/types" + "bk-bcs/bcs-services/bcs-network/qcloud-eip/conf" ) const ( // start eni route table id startENIRouteTableID = 100 + + mainRouteTableID = 254 ) // EIP object for use tencent network interface @@ -99,30 +102,16 @@ func (eip *EIP) Init(file string, eniNum int, ipNum int) { os.Exit(1) } - // if there is already some network interface name starting with netConf.ENIPrefix, - // it means someone has already initialized the network of this cvm, - // so stop the init action - for i := 0; i < eniNum; i++ { - eniName := fmt.Sprintf("%s%d", netConf.ENIPrefix, i) - blog.Infof("check eni %s......", eniName) - eniIPAddr, _, _ := getIPAddrByName(eniName) - if len(eniIPAddr) != 0 { - blog.Errorf("%s is existed, no need to init", eniName) - os.Exit(1) - } - } - // record route tables ids var routeTableIDs []int + // map for link name to route table id + routeTableIDMap := make(map[string]int) // record all available ips secondaryIPMap := make(map[string][]string) // according to the demanded network interface number and ip address number, // apply certain ip addresses for each newly applied network interface for i := 0; i < eniNum; i++ { - // for each newly applied network interface, create a route table in later steps, - // here we just calculate the route table id and record it in an array - tableID := startENIRouteTableID + i - routeTableIDs = append(routeTableIDs, tableID) + // for each eni, there is a IP num limitation // apply (limitation - 1) ip, because the default primary ip ipLimit := getMaxPrivateIPNumPerENI(int(*instance.CPU), int(*instance.Memory)) @@ -149,20 +138,40 @@ func (eip *EIP) Init(file string, eniNum int, ipNum int) { // record other applied ip addresses blog.Infof("set up eni %s ......", eniLocalInterfaceName) var secondaryIPs []string + var primaryIP string for _, privateIPObj := range newENI.PrivateIpAddressSet { if *privateIPObj.Primary { - err = setupNetworkInterface(*privateIPObj.PrivateIpAddress, *subnet.CidrBlock, eniLocalInterfaceName, *newENI.MacAddress) - if err != nil { - blog.Errorf("set up networkinterface %s with ip %s mac %s failed, err %s", eniLocalInterfaceName, *privateIPObj.PrivateIpAddress, *newENI.MacAddress, err.Error()) - os.Exit(1) - } - blog.Infof("set up eni with primary ip %s done", *privateIPObj.PrivateIpAddress) + primaryIP = *privateIPObj.PrivateIpAddress } else { blog.Infof("get secondary ip %s", *privateIPObj.PrivateIpAddress) secondaryIPs = append(secondaryIPs, *privateIPObj.PrivateIpAddress) } } secondaryIPMap[strings.ToLower(*newENI.MacAddress)] = secondaryIPs + + // if there is already some network interface name starting with netConf.ENIPrefix, + // it means someone has already initialized the network of this cvm, + // so stop the init action + linkName := fmt.Sprintf("%s%d", netConf.ENIPrefix, i) + blog.Infof("check eni %s......", linkName) + eniIPAddr, _, _ := getIPAddrByName(linkName) + if len(eniIPAddr) != 0 { + blog.Errorf("%s is existed, no need to set up", linkName) + continue + } + // set up eni + err = setupNetworkInterface(primaryIP, *subnet.CidrBlock, eniLocalInterfaceName, *newENI.MacAddress) + if err != nil { + blog.Errorf("set up networkinterface %s with ip %s mac %s failed, err %s", eniLocalInterfaceName, primaryIP, *newENI.MacAddress, err.Error()) + os.Exit(1) + } + blog.Infof("set up eni with primary ip %s done", primaryIP) + + // for each newly applied network interface, create a route table in later steps, + // here we just calculate the route table id and record it in an array + tableID := startENIRouteTableID + i + routeTableIDs = append(routeTableIDs, tableID) + routeTableIDMap[linkName] = tableID } // init network environment @@ -204,11 +213,10 @@ func (eip *EIP) Init(file string, eniNum int, ipNum int) { } gw := ip.NextIP(cidrIP) // add default route into each route tables - for index, id := range routeTableIDs { - linkName := fmt.Sprintf("%s%d", netConf.ENIPrefix, index) - link, err := netlink.LinkByName(linkName) + for eniName, id := range routeTableIDMap { + link, err := netlink.LinkByName(eniName) if err != nil { - blog.Errorf("get link by name %s failed, err %s", linkName, err.Error()) + blog.Errorf("get link by name %s failed, err %s", eniName, err.Error()) os.Exit(1) } defaultRouteRule := &netlink.Route{ @@ -224,6 +232,25 @@ func (eip *EIP) Init(file string, eniNum int, ipNum int) { blog.Errorf("add default route %v for table %d failed, err %s", defaultRouteRule, id, err.Error()) os.Exit(1) } + + // if bridgeName is not null, copy bridge route into each eni route table + if len(netConf.BridgeName) != 0 { + // get routes in main route table about bridge + bridgeRoutes, err := getBridgeRoutes(mainRouteTableID, netConf.BridgeName) + if err != nil { + blog.Warnf("get bridge %s routes failed, err %s", netConf.BridgeName) + continue + } + for _, r := range bridgeRoutes { + r.Table = id + blog.Info("add bridge route %+v for bridge %s", r, netConf.BridgeName) + err = netlink.RouteAdd(&r) + if err != nil { + blog.Errorf("add bridge route failed, err %s", err.Error()) + continue + } + } + } } // register ip pool for eni @@ -270,10 +297,28 @@ func (eip *EIP) Init(file string, eniNum int, ipNum int) { } blog.Infof("register ip pool to netservice done") + // get ip pool in netservice + existedPool, err := netSvcClient.GetPool(pool) + if err != nil { + blog.Errorf("get netservice pool failed, err %s", err.Error()) + os.Exit(1) + } + existedIPs := make(map[string]string) + for _, ip := range existedPool.Reserved { + existedIPs[ip] = ip + } + for _, ip := range existedPool.Active { + existedIPs[ip] = ip + } + // add newly ip addresses into ip pool in netservice blog.Infof("update ip instances to netservice") for mac, ips := range secondaryIPMap { for _, ip := range ips { + if _, ok := existedIPs[ip]; ok { + blog.Infof("ip %s already in pool, skip", ip) + continue + } ipIns := new(netsvc.IPInst) ipIns.IPAddr = ip ipIns.MacAddr = mac @@ -412,6 +457,24 @@ func (eip *EIP) Recover(file string, eniNum int) { blog.Errorf("add default route %v for table %d failed, err %s", defaultRouteRule, id, err.Error()) os.Exit(1) } + + // if bridgeName is not null, copy bridge route into each eni route table + if len(netConf.BridgeName) != 0 { + bridgeRoutes, err := getBridgeRoutes(mainRouteTableID, netConf.BridgeName) + if err != nil { + blog.Warnf("get bridge %s routes failed, err %s", netConf.BridgeName) + continue + } + for _, r := range bridgeRoutes { + r.Table = id + blog.Info("add bridge route %+v for bridge %s", r, netConf.BridgeName) + err = netlink.RouteAdd(&r) + if err != nil { + blog.Errorf("add bridge route failed, err %s", err.Error()) + continue + } + } + } } blog.Infof("congratulations, node recovery successfully!") @@ -555,6 +618,7 @@ func (eip *EIP) Release(file string) { err = eip.doDeregister(netConf) if err != nil { blog.Errorf("do deregister failed, err %s", err.Error()) + os.Exit(1) } // create cvm client diff --git a/bcs-services/bcs-network/qcloud-eip/eip/instance.go b/bcs-services/bcs-network/qcloud-eip/eip/instance.go index 21672e9a66..4c3c90c911 100644 --- a/bcs-services/bcs-network/qcloud-eip/eip/instance.go +++ b/bcs-services/bcs-network/qcloud-eip/eip/instance.go @@ -34,6 +34,12 @@ func newInstanceClient(conf *conf.NetConf) *instanceClient { conf.UUID, ) cpf := profile.NewClientProfile() + + // set tencentcloud domain + if len(conf.TencentCloudCVMDomain) != 0 { + cpf.HttpProfile.Endpoint = conf.TencentCloudCVMDomain + } + client, err := cvm.NewClient(credential, conf.Region, cpf) if err != nil { blog.Errorf("new instance client failed, err %s", err.Error()) diff --git a/bcs-services/bcs-network/qcloud-eip/eip/netservice.go b/bcs-services/bcs-network/qcloud-eip/eip/netservice.go index 85df6e1af7..11f07f6f6e 100644 --- a/bcs-services/bcs-network/qcloud-eip/eip/netservice.go +++ b/bcs-services/bcs-network/qcloud-eip/eip/netservice.go @@ -40,7 +40,7 @@ func NewNetSvcClient(conf *conf.NetArgs) (*NetSvcClient, error) { if len(conf.PubKey) == 0 && len(conf.Key) == 0 && len(conf.Ca) == 0 { client, clientErr = netservice.NewClient() } else { - client, clientErr = netservice.NewTLSClient(conf.Ca, conf.Key, conf.PubKey, static.ServerCertPwd) + client, clientErr = netservice.NewTLSClient(conf.Ca, conf.Key, conf.PubKey, static.ClientCertPwd) } if clientErr != nil { return nil, clientErr @@ -73,6 +73,15 @@ func (c *NetSvcClient) CreateOrUpdatePool(pool *netsvc.NetPool) error { return c.client.UpdatePool(pool) } +// GetPool get net pool +func (c *NetSvcClient) GetPool(pool *netsvc.NetPool) (*netsvc.NetPool, error) { + p, err := c.client.GetPool(pool.Cluster, pool.Net) + if err != nil { + return nil, err + } + return p[0], nil +} + // UpdateIPInstance update ip instance func (c *NetSvcClient) UpdateIPInstance(ins *netsvc.IPInst) error { err := c.client.UpdateIPInstance(ins) diff --git a/bcs-services/bcs-network/qcloud-eip/eip/util.go b/bcs-services/bcs-network/qcloud-eip/eip/util.go index d3884f62cb..d768d65b14 100644 --- a/bcs-services/bcs-network/qcloud-eip/eip/util.go +++ b/bcs-services/bcs-network/qcloud-eip/eip/util.go @@ -13,13 +13,48 @@ package eip import ( - "bk-bcs/bcs-common/common/blog" "fmt" "net" "strconv" "strings" + "golang.org/x/sys/unix" + + "github.com/vishvananda/netlink" + + "bk-bcs/bcs-common/common/blog" ) +func getBridgeRoutes(tableID int, bridgeName string) ([]netlink.Route, error) { + link, err := netlink.LinkByName(bridgeName) + if err != nil { + blog.Errorf("there is no bridge with name %s", bridgeName) + return nil, fmt.Errorf("there is no bridge with name %s", bridgeName) + } + addr, _, _ := getIPAddrByName(bridgeName) + if len(addr) == 0 { + blog.Errorf("get bridge %s addr failed", bridgeName) + return nil, fmt.Errorf("get bridge %s addr failed", bridgeName) + } + ipObj := net.ParseIP(addr) + if ipObj == nil { + blog.Errorf("parse ip %s return nil", addr) + return nil, fmt.Errorf("parse ip %s return nil", addr) + } + bridgeRouteRule := netlink.Route{ + LinkIndex: link.Attrs().Index, + Scope: netlink.SCOPE_LINK, + Src: ipObj, + Table: tableID, + } + routes, err := netlink.RouteListFiltered(unix.AF_INET, &bridgeRouteRule, + netlink.RT_FILTER_TABLE|netlink.RT_FILTER_SCOPE|netlink.RT_FILTER_OIF|netlink.RT_FILTER_SRC) + if err != nil { + blog.Errorf("failed to list route list with route %+v , err %s", bridgeRouteRule, err.Error()) + return nil, err + } + return routes, nil +} + //GetIPAddrByName return eni ip address, mask, mac address func getIPAddrByName(name string) (string, int, string) { //get ip address diff --git a/bcs-services/bcs-network/qcloud-eip/eip/vpc.go b/bcs-services/bcs-network/qcloud-eip/eip/vpc.go index 161bcabb9b..b95cc0eae2 100644 --- a/bcs-services/bcs-network/qcloud-eip/eip/vpc.go +++ b/bcs-services/bcs-network/qcloud-eip/eip/vpc.go @@ -55,6 +55,12 @@ func newVPCClient(conf *conf.NetConf, vpcID string) *vpcClient { conf.UUID, ) cpf := profile.NewClientProfile() + + // set tencentcloud domain + if len(conf.TencentCloudVPCDomain) != 0 { + cpf.HttpProfile.Endpoint = conf.TencentCloudVPCDomain + } + client, err := vpc.NewClient(credential, conf.Region, cpf) if err != nil { blog.Errorf("new vpc client failed, err %s", err.Error()) From 1e79253272fe5e277782ca58668c9a25197354d4 Mon Sep 17 00:00:00 2001 From: abstractmj Date: Mon, 6 Jul 2020 14:55:51 +0800 Subject: [PATCH 2/2] fix: rm bridge route, add rule with priority 2048; issue #515 --- .../bcs-network/qcloud-eip/conf/conf.go | 4 -- .../bcs-network/qcloud-eip/eip/eip.go | 39 +------------------ 2 files changed, 2 insertions(+), 41 deletions(-) diff --git a/bcs-services/bcs-network/qcloud-eip/conf/conf.go b/bcs-services/bcs-network/qcloud-eip/conf/conf.go index 889ef9adb7..da14012eb1 100644 --- a/bcs-services/bcs-network/qcloud-eip/conf/conf.go +++ b/bcs-services/bcs-network/qcloud-eip/conf/conf.go @@ -45,10 +45,6 @@ type NetConf struct { UUID string `json:"uuid"` SubnetID string `json:"subnetId,omitempty"` MTU int `json:"mtu,omitempty"` - // Design: If both bridge network mode and elastic network card mode exist - // copy route to bridge into each eni route table, otherwise pod with eni can - // not communicate with the bridge-mode pod - BridgeName string `json:"bridgeName"` NetService *NetArgs `json:"netservice,omitempty"` Args *bcsconf.CNIArgs } diff --git a/bcs-services/bcs-network/qcloud-eip/eip/eip.go b/bcs-services/bcs-network/qcloud-eip/eip/eip.go index cc4828c66a..e87d4b8f79 100644 --- a/bcs-services/bcs-network/qcloud-eip/eip/eip.go +++ b/bcs-services/bcs-network/qcloud-eip/eip/eip.go @@ -232,25 +232,6 @@ func (eip *EIP) Init(file string, eniNum int, ipNum int) { blog.Errorf("add default route %v for table %d failed, err %s", defaultRouteRule, id, err.Error()) os.Exit(1) } - - // if bridgeName is not null, copy bridge route into each eni route table - if len(netConf.BridgeName) != 0 { - // get routes in main route table about bridge - bridgeRoutes, err := getBridgeRoutes(mainRouteTableID, netConf.BridgeName) - if err != nil { - blog.Warnf("get bridge %s routes failed, err %s", netConf.BridgeName) - continue - } - for _, r := range bridgeRoutes { - r.Table = id - blog.Info("add bridge route %+v for bridge %s", r, netConf.BridgeName) - err = netlink.RouteAdd(&r) - if err != nil { - blog.Errorf("add bridge route failed, err %s", err.Error()) - continue - } - } - } } // register ip pool for eni @@ -457,24 +438,6 @@ func (eip *EIP) Recover(file string, eniNum int) { blog.Errorf("add default route %v for table %d failed, err %s", defaultRouteRule, id, err.Error()) os.Exit(1) } - - // if bridgeName is not null, copy bridge route into each eni route table - if len(netConf.BridgeName) != 0 { - bridgeRoutes, err := getBridgeRoutes(mainRouteTableID, netConf.BridgeName) - if err != nil { - blog.Warnf("get bridge %s routes failed, err %s", netConf.BridgeName) - continue - } - for _, r := range bridgeRoutes { - r.Table = id - blog.Info("add bridge route %+v for bridge %s", r, netConf.BridgeName) - err = netlink.RouteAdd(&r) - if err != nil { - blog.Errorf("add bridge route failed, err %s", err.Error()) - continue - } - } - } } blog.Infof("congratulations, node recovery successfully!") @@ -752,6 +715,7 @@ func configureHostNS(hostIfName string, ipNet *net.IPNet, routeTableID int) erro ruleToTable := netlink.NewRule() ruleToTable.Dst = ipNet ruleToTable.Table = routeTableID + ruleToTable.Priority = 2048 err = netlink.RuleDel(ruleToTable) if err != nil { blog.Warnf("clean old rule to table %s failed, err %s", ruleToTable.String(), err.Error()) @@ -765,6 +729,7 @@ func configureHostNS(hostIfName string, ipNet *net.IPNet, routeTableID int) erro ruleFromTaskgroup := netlink.NewRule() ruleFromTaskgroup.Src = ipNet ruleFromTaskgroup.Table = routeTableID + ruleFromTaskgroup.Priority = 2048 err = netlink.RuleDel(ruleFromTaskgroup) if err != nil { blog.Warnf("clean old rule from taskgroup %s failed, err %s", ruleToTable.String(), err.Error())