diff --git a/go.mod b/go.mod index 2652578..37fd83c 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( cloud.google.com/go/compute/metadata v0.5.2 github.com/Mellanox/rdmamap v1.1.0 github.com/containerd/nri v0.8.0 - github.com/containernetworking/plugins v1.6.1 github.com/vishvananda/netlink v1.3.0 + github.com/vishvananda/netns v0.0.5 golang.org/x/sys v0.28.0 golang.org/x/time v0.8.0 k8s.io/api v0.31.3 @@ -42,10 +42,11 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/onsi/ginkgo/v2 v2.22.0 // indirect + github.com/onsi/gomega v1.36.0 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/vishvananda/netns v0.0.5 // indirect github.com/x448/float16 v0.8.4 // indirect golang.org/x/net v0.32.0 // indirect golang.org/x/oauth2 v0.24.0 // indirect diff --git a/go.sum b/go.sum index ffc0c0a..d744ba2 100644 --- a/go.sum +++ b/go.sum @@ -8,10 +8,6 @@ github.com/containerd/nri v0.8.0 h1:n1S753B9lX8RFrHYeSgwVvS1yaUcHjxbB+f+xzEncRI= github.com/containerd/nri v0.8.0/go.mod h1:uSkgBrCdEtAiEz4vnrq8gmAC4EnVAM5Klt0OuK5rZYQ= github.com/containerd/ttrpc v1.2.6 h1:zG+Kn5EZ6MUYCS1t2Hmt2J4tMVaLSFEJVOraDQwNPC4= github.com/containerd/ttrpc v1.2.6/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= -github.com/containernetworking/cni v1.2.3 h1:hhOcjNVUQTnzdRJ6alC5XF+wd9mfGIUaj8FuJbEslXM= -github.com/containernetworking/cni v1.2.3/go.mod h1:DuLgF+aPd3DzcTQTtp/Nvl1Kim23oFKdm2okJzBQA5M= -github.com/containernetworking/plugins v1.6.1 h1:bYd2bpE6hEBqexyaiI2/sst0xJ+v7pEMWrjA5qtkxiU= -github.com/containernetworking/plugins v1.6.1/go.mod h1:SP5UG3jDO9LtmfbBJdP+nl3A1atOtbj2MBOYsnaxy64= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= diff --git a/pkg/driver/driver.go b/pkg/driver/driver.go index 644ad72..98f2593 100644 --- a/pkg/driver/driver.go +++ b/pkg/driver/driver.go @@ -188,10 +188,6 @@ func (np *NetworkDriver) RunPodSandbox(ctx context.Context, pod *api.PodSandbox) // get the pod network namespace ns := getNetworkNamespace(pod) // host network pods are skipped - if ns == "" { - return nil - } - // host network pods are skipped if ns == "" { klog.V(2).Infof("RunPodSandbox pod %s/%s using host network, skipping", pod.Namespace, pod.Name) return nil @@ -202,18 +198,34 @@ func (np *NetworkDriver) RunPodSandbox(ctx context.Context, pod *api.PodSandbox) if config.Opaque == nil { continue } - klog.V(4).Infof("podStartHook Configuration %s", config.Opaque.Parameters.String()) + klog.V(4).Infof("podStartHook Configuration %s", string(config.Opaque.Parameters.String())) // TODO get config options here, it can add ips or commands // to add routes, run dhcp, rename the interface ... whatever } + // Process the configurations of the ResourceClaim for _, result := range allocation.Devices.Results { if result.Driver != np.driverName { continue } - // TODO see https://github.com/containernetworking/plugins/tree/main/plugins/main - // for better examples of low level implementations using netlink for more complex - // scenarios like host-device, ipvlan, macvlan, ... + klog.Infof("RunPodSandbox allocation.Devices.Result: %#v", result) + // TODO signal this via DRA + if rdmaDev, _ := rdmamap.GetRdmaDeviceForNetdevice(result.Device); rdmaDev != "" { + err := nsAttachRdmadev(rdmaDev, ns) + if err != nil { + klog.Infof("RunPodSandbox error getting RDMA device %s to namespace %s: %v", result.Device, ns, err) + continue + } + } + + // TODO config options to rename the device and pass parameters + // use https://github.com/opencontainers/runtime-spec/pull/1271 + err := nsAttachNetdev(result.Device, ns, result.Device) + if err != nil { + klog.Infof("RunPodSandbox error moving device %s to namespace %s: %v", result.Device, ns, err) + return err + } + } return nil } @@ -229,8 +241,8 @@ func (np *NetworkDriver) StopPodSandbox(ctx context.Context, pod *api.PodSandbox // get the pod network namespace ns := getNetworkNamespace(pod) - // host network pods are skipped if ns == "" { + klog.V(2).Infof("StopPodSandbox pod %s/%s using host network, skipping", pod.Namespace, pod.Name) return nil } @@ -238,7 +250,7 @@ func (np *NetworkDriver) StopPodSandbox(ctx context.Context, pod *api.PodSandbox if config.Opaque == nil { continue } - klog.V(4).Infof("podStopHook Configuration %s", config.Opaque.Parameters.String()) + klog.V(4).Infof("podStopHook Configuration %s", string(config.Opaque.Parameters.String())) // TODO get config options here, it can add ips or commands // to add routes, run dhcp, rename the interface ... whatever } @@ -248,8 +260,24 @@ func (np *NetworkDriver) StopPodSandbox(ctx context.Context, pod *api.PodSandbox continue } klog.V(4).Infof("podStopHook Device %s", result.Device) - // TODO get config options here, it can add ips or commands - // to add routes, run dhcp, rename the interface ... whatever + // TODO config options to rename the device and pass parameters + // use https://github.com/opencontainers/runtime-spec/pull/1271 + err := nsDetachNetdev(ns, result.Device) + if err != nil { + klog.Infof("RunPodSandbox error moving device %s to namespace %s: %v", result.Device, ns, err) + continue + } + } + return nil +} + +func (np *NetworkDriver) RemovePodSandbox(_ context.Context, pod *api.PodSandbox) error { + klog.V(2).Infof("RemovePodSandbox pod %s/%s: ips=%v", pod.GetNamespace(), pod.GetName(), pod.GetIps()) + // get the pod network namespace + ns := getNetworkNamespace(pod) + if ns == "" { + klog.V(2).Infof("RemovePodSandbox pod %s/%s using host network, skipping", pod.Namespace, pod.Name) + return nil } return nil } @@ -356,10 +384,9 @@ func (np *NetworkDriver) PublishResources(ctx context.Context) { device.Basic.Attributes["sriov_vfs"] = resourceapi.DeviceAttribute{IntValue: &vfs} } resources.Devices = append(resources.Devices, device) - + klog.V(4).Infof("Found following network interfaces %s", iface.Name) } - klog.V(4).Infof("Found following network interfaces %#v", resources.Devices) if len(resources.Devices) > 0 { np.draPlugin.PublishResources(ctx, resources) } @@ -387,7 +414,7 @@ func (np *NetworkDriver) NodePrepareResources(ctx context.Context, request *drap } for _, claimReq := range request.GetClaims() { - klog.V(2).Infof("NodePrepareResources: Claim Request %#v", claimReq) + klog.V(2).Infof("NodePrepareResources: Claim Request %s/%s", claimReq.Namespace, claimReq.Name) devices, err := np.nodePrepareResource(ctx, claimReq) if err != nil { resp.Claims[claimReq.UID] = &drapb.NodePrepareResourceResponse{ @@ -483,7 +510,6 @@ func (np *NetworkDriver) nodeUnprepareResource(ctx context.Context, claimReq *dr } defer np.claimAllocations.Remove(types.UID(claimReq.UID)) klog.Infof("claim %s/%s with allocation %#v", claimReq.Namespace, claimReq.Name, allocation) - // TODO do unpreparing things return nil } diff --git a/pkg/driver/hostdevice.go b/pkg/driver/hostdevice.go index 7c27e71..0cb051b 100644 --- a/pkg/driver/hostdevice.go +++ b/pkg/driver/hostdevice.go @@ -19,12 +19,11 @@ package driver import ( "fmt" - "github.com/containernetworking/plugins/pkg/ns" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" ) -func MoveLinkIn(hostIfName string, containerNsPAth string, ifName string) error { +func nsAttachNetdev(hostIfName string, containerNsPAth string, ifName string) error { hostDev, err := netlink.LinkByName(hostIfName) if err != nil { return err @@ -43,11 +42,11 @@ func MoveLinkIn(hostIfName string, containerNsPAth string, ifName string) error return fmt.Errorf("failed to set %q down: %v", hostDev.Attrs().Name, err) } - containerNs, err := ns.GetNS(containerNsPAth) + containerNs, err := netns.GetFromPath(containerNsPAth) if err != nil { return err } - attrs.Namespace = containerNs.Fd() + attrs.Namespace = netlink.NsFd(containerNs) dev := &netlink.Device{ LinkAttrs: attrs, @@ -60,7 +59,7 @@ func MoveLinkIn(hostIfName string, containerNsPAth string, ifName string) error // to avoid golang problem with goroutines we create the socket in the // namespace and use it directly - nhNs, err := netlink.NewHandleAt(netns.NsHandle(containerNs.Fd())) + nhNs, err := netlink.NewHandleAt(containerNs) if err != nil { return err } @@ -78,20 +77,14 @@ func MoveLinkIn(hostIfName string, containerNsPAth string, ifName string) error return nil } -func MoveLinkOut(containerNsPAth string, devName string) error { - defaultNs, err := ns.GetCurrentNS() - if err != nil { - return err - } - defer defaultNs.Close() - - ns, err := netns.GetFromPath(containerNsPAth) +func nsDetachNetdev(containerNsPAth string, devName string) error { + containerNs, err := netns.GetFromPath(containerNsPAth) if err != nil { return fmt.Errorf("could not get network namespace from path %s for network device %s : %w", containerNsPAth, devName, err) } // to avoid golang problem with goroutines we create the socket in the // namespace and use it directly - nhNs, err := netlink.NewHandleAt(ns) + nhNs, err := netlink.NewHandleAt(containerNs) if err != nil { return fmt.Errorf("could not get network namespace handle: %w", err) } @@ -122,7 +115,7 @@ func MoveLinkOut(containerNsPAth string, devName string) error { } defer rootNs.Close() - attrs.Namespace = int(netlink.NsFd(rootNs)) + attrs.Namespace = netlink.NsFd(rootNs) dev := &netlink.Device{ LinkAttrs: attrs, diff --git a/pkg/driver/rdmadevice.go b/pkg/driver/rdmadevice.go index 665fc3f..e79b761 100644 --- a/pkg/driver/rdmadevice.go +++ b/pkg/driver/rdmadevice.go @@ -19,56 +19,58 @@ package driver import ( "fmt" - "github.com/containernetworking/plugins/pkg/ns" "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" ) // Based on existing RDMA CNI plugin // https://github.com/k8snetworkplumbingwg/rdma-cni -func MoveRDMALinkIn(hostIfName string, containerNsPAth string) error { - containerNs, err := ns.GetNS(containerNsPAth) +func nsAttachRdmadev(hostIfName string, containerNsPAth string) error { + containerNs, err := netns.GetFromPath(containerNsPAth) if err != nil { - return err + return fmt.Errorf("could not get network namespace from path %s for network device %s : %w", containerNsPAth, hostIfName, err) } + hostDev, err := netlink.RdmaLinkByName(hostIfName) if err != nil { return err } - if err = netlink.RdmaLinkSetNsFd(hostDev, uint32(containerNs.Fd())); err != nil { + if err = netlink.RdmaLinkSetNsFd(hostDev, uint32(containerNs)); err != nil { return fmt.Errorf("failed to move %q to container ns: %v", hostDev.Attrs.Name, err) } return nil } -func MoveRDMALinkOut(containerNsPAth string, ifName string) error { - containerNs, err := ns.GetNS(containerNsPAth) +func nsDetachRdmadev(containerNsPAth string, ifName string) error { + containerNs, err := netns.GetFromPath(containerNsPAth) if err != nil { - return err + return fmt.Errorf("could not get network namespace from path %s for network device %s : %w", containerNsPAth, ifName, err) } - defaultNs, err := ns.GetCurrentNS() + + // to avoid golang problem with goroutines we create the socket in the + // namespace and use it directly + nhNs, err := netlink.NewHandleAt(containerNs) if err != nil { - return err + return fmt.Errorf("could not get network namespace handle: %w", err) } - defer defaultNs.Close() - err = containerNs.Do(func(_ ns.NetNS) error { - dev, err := netlink.RdmaLinkByName(ifName) - if err != nil { - return fmt.Errorf("failed to find %q: %v", ifName, err) - } - - if err = netlink.RdmaLinkSetNsFd(dev, uint32(defaultNs.Fd())); err != nil { - return fmt.Errorf("failed to move %q to host netns: %v", dev.Attrs.Name, err) - } - return nil - }) + dev, err := nhNs.RdmaLinkByName(ifName) + if err != nil { + return fmt.Errorf("failed to find %q: %v", ifName, err) + } + rootNs, err := netns.Get() if err != nil { return err } + defer rootNs.Close() + if err = nhNs.RdmaLinkSetNsFd(dev, uint32(rootNs)); err != nil { + return fmt.Errorf("failed to move %q to host netns: %v", dev.Attrs.Name, err) + } return nil + }