fix namespace plumbering logic

Change-Id: I9c55046bb218819beb498a5c9badf1f6ce96a352

go.mod

@@ -6,8 +6,8 @@ require (
 	cloud.google.com/go/compute/metadata v0.5.2
 	github.com/Mellanox/rdmamap v1.1.0
 	github.com/containerd/nri v0.8.0
-	github.com/containernetworking/plugins v1.6.1
 	github.com/vishvananda/netlink v1.3.0
+	github.com/vishvananda/netns v0.0.5
 	golang.org/x/sys v0.28.0
 	golang.org/x/time v0.8.0
 	k8s.io/api v0.31.3
@@ -42,10 +42,11 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/onsi/ginkgo/v2 v2.22.0 // indirect
+	github.com/onsi/gomega v1.36.0 // indirect
 	github.com/opencontainers/runtime-spec v1.2.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/vishvananda/netns v0.0.5 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	golang.org/x/net v0.32.0 // indirect
 	golang.org/x/oauth2 v0.24.0 // indirect

go.sum

@@ -8,10 +8,6 @@ github.com/containerd/nri v0.8.0 h1:n1S753B9lX8RFrHYeSgwVvS1yaUcHjxbB+f+xzEncRI=
 github.com/containerd/nri v0.8.0/go.mod h1:uSkgBrCdEtAiEz4vnrq8gmAC4EnVAM5Klt0OuK5rZYQ=
 github.com/containerd/ttrpc v1.2.6 h1:zG+Kn5EZ6MUYCS1t2Hmt2J4tMVaLSFEJVOraDQwNPC4=
 github.com/containerd/ttrpc v1.2.6/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o=
-github.com/containernetworking/cni v1.2.3 h1:hhOcjNVUQTnzdRJ6alC5XF+wd9mfGIUaj8FuJbEslXM=
-github.com/containernetworking/cni v1.2.3/go.mod h1:DuLgF+aPd3DzcTQTtp/Nvl1Kim23oFKdm2okJzBQA5M=
-github.com/containernetworking/plugins v1.6.1 h1:bYd2bpE6hEBqexyaiI2/sst0xJ+v7pEMWrjA5qtkxiU=
-github.com/containernetworking/plugins v1.6.1/go.mod h1:SP5UG3jDO9LtmfbBJdP+nl3A1atOtbj2MBOYsnaxy64=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=

pkg/driver/driver.go

@@ -188,10 +188,6 @@ func (np *NetworkDriver) RunPodSandbox(ctx context.Context, pod *api.PodSandbox)
 	// get the pod network namespace
 	ns := getNetworkNamespace(pod)
 	// host network pods are skipped
-	if ns == "" {
-		return nil
-	}
-	// host network pods are skipped
 	if ns == "" {
 		klog.V(2).Infof("RunPodSandbox pod %s/%s using host network, skipping", pod.Namespace, pod.Name)
 		return nil
@@ -202,18 +198,34 @@ func (np *NetworkDriver) RunPodSandbox(ctx context.Context, pod *api.PodSandbox)
 		if config.Opaque == nil {
 			continue
 		}
-		klog.V(4).Infof("podStartHook Configuration %s", config.Opaque.Parameters.String())
+		klog.V(4).Infof("podStartHook Configuration %s", string(config.Opaque.Parameters.String()))
 		// TODO get config options here, it can add ips or commands
 		// to add routes, run dhcp, rename the interface ... whatever
 	}
+
 	// Process the configurations of the ResourceClaim
 	for _, result := range allocation.Devices.Results {
 		if result.Driver != np.driverName {
 			continue
 		}
-		// TODO see https://github.com/containernetworking/plugins/tree/main/plugins/main
-		// for better examples of low level implementations using netlink for more complex
-		// scenarios like host-device, ipvlan, macvlan, ...
+		klog.Infof("RunPodSandbox allocation.Devices.Result: %#v", result)
+		// TODO signal this via DRA
+		if rdmaDev, _ := rdmamap.GetRdmaDeviceForNetdevice(result.Device); rdmaDev != "" {
+			err := nsAttachRdmadev(rdmaDev, ns)
+			if err != nil {
+				klog.Infof("RunPodSandbox error getting RDMA device %s to namespace %s: %v", result.Device, ns, err)
+				continue
+			}
+		}
+
+		// TODO config options to rename the device and pass parameters
+		// use https://github.com/opencontainers/runtime-spec/pull/1271
+		err := nsAttachNetdev(result.Device, ns, result.Device)
+		if err != nil {
+			klog.Infof("RunPodSandbox error moving device %s to namespace %s: %v", result.Device, ns, err)
+			return err
+		}
+
 	}
 	return nil
 }
@@ -229,16 +241,16 @@ func (np *NetworkDriver) StopPodSandbox(ctx context.Context, pod *api.PodSandbox
 
 	// get the pod network namespace
 	ns := getNetworkNamespace(pod)
-	// host network pods are skipped
 	if ns == "" {
+		klog.V(2).Infof("StopPodSandbox pod %s/%s using host network, skipping", pod.Namespace, pod.Name)
 		return nil
 	}
 
 	for _, config := range allocation.Devices.Config {
 		if config.Opaque == nil {
 			continue
 		}
-		klog.V(4).Infof("podStopHook Configuration %s", config.Opaque.Parameters.String())
+		klog.V(4).Infof("podStopHook Configuration %s", string(config.Opaque.Parameters.String()))
 		// TODO get config options here, it can add ips or commands
 		// to add routes, run dhcp, rename the interface ... whatever
 	}
@@ -248,8 +260,24 @@ func (np *NetworkDriver) StopPodSandbox(ctx context.Context, pod *api.PodSandbox
 			continue
 		}
 		klog.V(4).Infof("podStopHook Device %s", result.Device)
-		// TODO get config options here, it can add ips or commands
-		// to add routes, run dhcp, rename the interface ... whatever
+		// TODO config options to rename the device and pass parameters
+		// use https://github.com/opencontainers/runtime-spec/pull/1271
+		err := nsDetachNetdev(ns, result.Device)
+		if err != nil {
+			klog.Infof("RunPodSandbox error moving device %s to namespace %s: %v", result.Device, ns, err)
+			continue
+		}
+	}
+	return nil
+}
+
+func (np *NetworkDriver) RemovePodSandbox(_ context.Context, pod *api.PodSandbox) error {
+	klog.V(2).Infof("RemovePodSandbox pod %s/%s: ips=%v", pod.GetNamespace(), pod.GetName(), pod.GetIps())
+	// get the pod network namespace
+	ns := getNetworkNamespace(pod)
+	if ns == "" {
+		klog.V(2).Infof("RemovePodSandbox pod %s/%s using host network, skipping", pod.Namespace, pod.Name)
+		return nil
 	}
 	return nil
 }
@@ -356,10 +384,9 @@ func (np *NetworkDriver) PublishResources(ctx context.Context) {
 				device.Basic.Attributes["sriov_vfs"] = resourceapi.DeviceAttribute{IntValue: &vfs}
 			}
 			resources.Devices = append(resources.Devices, device)
-
+			klog.V(4).Infof("Found following network interfaces %s", iface.Name)
 		}
 
-		klog.V(4).Infof("Found following network interfaces %#v", resources.Devices)
 		if len(resources.Devices) > 0 {
 			np.draPlugin.PublishResources(ctx, resources)
 		}
@@ -387,7 +414,7 @@ func (np *NetworkDriver) NodePrepareResources(ctx context.Context, request *drap
 	}
 
 	for _, claimReq := range request.GetClaims() {
-		klog.V(2).Infof("NodePrepareResources: Claim Request %#v", claimReq)
+		klog.V(2).Infof("NodePrepareResources: Claim Request %s/%s", claimReq.Namespace, claimReq.Name)
 		devices, err := np.nodePrepareResource(ctx, claimReq)
 		if err != nil {
 			resp.Claims[claimReq.UID] = &drapb.NodePrepareResourceResponse{
@@ -483,7 +510,6 @@ func (np *NetworkDriver) nodeUnprepareResource(ctx context.Context, claimReq *dr
 	}
 	defer np.claimAllocations.Remove(types.UID(claimReq.UID))
 	klog.Infof("claim %s/%s with allocation %#v", claimReq.Namespace, claimReq.Name, allocation)
-	// TODO do unpreparing things
 	return nil
 }
 

pkg/driver/hostdevice.go

@@ -19,12 +19,11 @@ package driver
 import (
 	"fmt"
 
-	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/vishvananda/netlink"
 	"github.com/vishvananda/netns"
 )
 
-func MoveLinkIn(hostIfName string, containerNsPAth string, ifName string) error {
+func nsAttachNetdev(hostIfName string, containerNsPAth string, ifName string) error {
 	hostDev, err := netlink.LinkByName(hostIfName)
 	if err != nil {
 		return err
@@ -43,11 +42,11 @@ func MoveLinkIn(hostIfName string, containerNsPAth string, ifName string) error
 		return fmt.Errorf("failed to set %q down: %v", hostDev.Attrs().Name, err)
 	}
 
-	containerNs, err := ns.GetNS(containerNsPAth)
+	containerNs, err := netns.GetFromPath(containerNsPAth)
 	if err != nil {
 		return err
 	}
-	attrs.Namespace = containerNs.Fd()
+	attrs.Namespace = netlink.NsFd(containerNs)
 
 	dev := &netlink.Device{
 		LinkAttrs: attrs,
@@ -60,7 +59,7 @@ func MoveLinkIn(hostIfName string, containerNsPAth string, ifName string) error
 
 	// to avoid golang problem with goroutines we create the socket in the
 	// namespace and use it directly
-	nhNs, err := netlink.NewHandleAt(netns.NsHandle(containerNs.Fd()))
+	nhNs, err := netlink.NewHandleAt(containerNs)
 	if err != nil {
 		return err
 	}
@@ -78,20 +77,14 @@ func MoveLinkIn(hostIfName string, containerNsPAth string, ifName string) error
 	return nil
 }
 
-func MoveLinkOut(containerNsPAth string, devName string) error {
-	defaultNs, err := ns.GetCurrentNS()
-	if err != nil {
-		return err
-	}
-	defer defaultNs.Close()
-
-	ns, err := netns.GetFromPath(containerNsPAth)
+func nsDetachNetdev(containerNsPAth string, devName string) error {
+	containerNs, err := netns.GetFromPath(containerNsPAth)
 	if err != nil {
 		return fmt.Errorf("could not get network namespace from path %s for network device %s : %w", containerNsPAth, devName, err)
 	}
 	// to avoid golang problem with goroutines we create the socket in the
 	// namespace and use it directly
-	nhNs, err := netlink.NewHandleAt(ns)
+	nhNs, err := netlink.NewHandleAt(containerNs)
 	if err != nil {
 		return fmt.Errorf("could not get network namespace handle: %w", err)
 	}
@@ -122,7 +115,7 @@ func MoveLinkOut(containerNsPAth string, devName string) error {
 	}
 	defer rootNs.Close()
 
-	attrs.Namespace = int(netlink.NsFd(rootNs))
+	attrs.Namespace = netlink.NsFd(rootNs)
 
 	dev := &netlink.Device{
 		LinkAttrs: attrs,

pkg/driver/rdmadevice.go

@@ -19,56 +19,58 @@ package driver
 import (
 	"fmt"
 
-	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netns"
 )
 
 // Based on existing RDMA CNI plugin
 // https://github.com/k8snetworkplumbingwg/rdma-cni
 
-func MoveRDMALinkIn(hostIfName string, containerNsPAth string) error {
-	containerNs, err := ns.GetNS(containerNsPAth)
+func nsAttachRdmadev(hostIfName string, containerNsPAth string) error {
+	containerNs, err := netns.GetFromPath(containerNsPAth)
 	if err != nil {
-		return err
+		return fmt.Errorf("could not get network namespace from path %s for network device %s : %w", containerNsPAth, hostIfName, err)
 	}
+
 	hostDev, err := netlink.RdmaLinkByName(hostIfName)
 	if err != nil {
 		return err
 	}
 
-	if err = netlink.RdmaLinkSetNsFd(hostDev, uint32(containerNs.Fd())); err != nil {
+	if err = netlink.RdmaLinkSetNsFd(hostDev, uint32(containerNs)); err != nil {
 		return fmt.Errorf("failed to move %q to container ns: %v", hostDev.Attrs.Name, err)
 	}
 
 	return nil
 }
 
-func MoveRDMALinkOut(containerNsPAth string, ifName string) error {
-	containerNs, err := ns.GetNS(containerNsPAth)
+func nsDetachRdmadev(containerNsPAth string, ifName string) error {
+	containerNs, err := netns.GetFromPath(containerNsPAth)
 	if err != nil {
-		return err
+		return fmt.Errorf("could not get network namespace from path %s for network device %s : %w", containerNsPAth, ifName, err)
 	}
-	defaultNs, err := ns.GetCurrentNS()
+
+	// to avoid golang problem with goroutines we create the socket in the
+	// namespace and use it directly
+	nhNs, err := netlink.NewHandleAt(containerNs)
 	if err != nil {
-		return err
+		return fmt.Errorf("could not get network namespace handle: %w", err)
 	}
-	defer defaultNs.Close()
 
-	err = containerNs.Do(func(_ ns.NetNS) error {
-		dev, err := netlink.RdmaLinkByName(ifName)
-		if err != nil {
-			return fmt.Errorf("failed to find %q: %v", ifName, err)
-		}
-
-		if err = netlink.RdmaLinkSetNsFd(dev, uint32(defaultNs.Fd())); err != nil {
-			return fmt.Errorf("failed to move %q to host netns: %v", dev.Attrs.Name, err)
-		}
-		return nil
-	})
+	dev, err := nhNs.RdmaLinkByName(ifName)
+	if err != nil {
+		return fmt.Errorf("failed to find %q: %v", ifName, err)
+	}
 
+	rootNs, err := netns.Get()
 	if err != nil {
 		return err
 	}
+	defer rootNs.Close()
 
+	if err = nhNs.RdmaLinkSetNsFd(dev, uint32(rootNs)); err != nil {
+		return fmt.Errorf("failed to move %q to host netns: %v", dev.Attrs.Name, err)
+	}
 	return nil
+
 }