tetragon: Allow persistent enforcement during tetragon restart

cmd/tetragon/main.go

@@ -157,6 +157,34 @@ func stopProfile() {
 	}
 }
 
+func getOldBpfDir(path string) (string, error) {
+	if _, err := os.Stat(path); err != nil {
+		return "", nil
+	}
+	old := path + "_old"
+	// remove the 'xxx_old' leftover if neded
+	if _, err := os.Stat(old); err == nil {
+		os.RemoveAll(old)
+		log.Info("Found bpf leftover instance, removing: %s", old)
+	}
+	if err := os.Rename(path, old); err != nil {
+		return "", err
+	}
+	log.Infof("Found bpf instance: %s, moved to: %s", path, old)
+	return old, nil
+}
+
+func deleteOldBpfDir(path string) {
+	if path == "" {
+		return
+	}
+	if err := os.RemoveAll(path); err != nil {
+		log.Errorf("Failed to remove old bpf instance '%s': %s\n", path, err)
+		return
+	}
+	log.Infof("Removed bpf instance: %s", path)
+}
+
 func tetragonExecute() error {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -208,8 +236,6 @@ func tetragonExecute() error {
 	}
 	defer pidfile.Delete()
 
-	log.Info("BPF detected features: ", bpf.LogFeatures())
-
 	if option.Config.ForceLargeProgs && option.Config.ForceSmallProgs {
 		log.Fatalf("Can't specify --force-small-progs and --force-large-progs together")
 	}
@@ -222,6 +248,10 @@ func tetragonExecute() error {
 		log.Info("Force loading smallprograms")
 	}
 
+	if option.Config.KeepSensorsOnExit {
+		log.Info("Not unloading sensors on exit")
+	}
+
 	if viper.IsSet(option.KeyNetnsDir) {
 		defaults.NetnsDir = viper.GetString(option.KeyNetnsDir)
 	}
@@ -244,6 +274,15 @@ func tetragonExecute() error {
 	bpf.CheckOrMountFS("")
 	bpf.CheckOrMountDebugFS()
 	bpf.CheckOrMountCgroup2()
+	bpf.SetMapPrefix(option.Config.BpfDir)
+
+	oldBpfDir, err := getOldBpfDir(bpf.MapPrefixPath())
+	if err != nil {
+		return fmt.Errorf("Failed to move old tetragon base directory: %w", err)
+	}
+
+	// we need file system mounts setup above before we detect features
+	log.Info("BPF detected features: ", bpf.LogFeatures())
 
 	if option.Config.PprofAddr != "" {
 		go func() {
@@ -477,6 +516,8 @@ func tetragonExecute() error {
 		}
 	}
 
+	deleteOldBpfDir(oldBpfDir)
+
 	// k8s should have metrics, so periodically log only in a non k8s
 	if !option.Config.EnableK8s {
 		go logStatus(ctx, obs)

go.mod

@@ -8,7 +8,7 @@ toolchain go1.22.3
 require (
 	github.com/bombsimon/logrusr/v4 v4.1.0
 	github.com/cilium/cilium v1.15.6
-	github.com/cilium/ebpf v0.15.0
+	github.com/cilium/ebpf v0.15.1-0.20240703142256-12aca1027ee8
 	github.com/cilium/little-vm-helper v0.0.18
 	github.com/cilium/lumberjack/v2 v2.3.0
 	github.com/cilium/tetragon/api v0.0.0-00010101000000-000000000000

go.sum

@@ -55,8 +55,8 @@ github.com/cilium/cilium v1.15.6 h1:YT6UYuvdua6N1KQ6mRprymCct6Ee7uCE1hckbAR2bRM=
 github.com/cilium/cilium v1.15.6/go.mod h1:UEP0tpPVhdrLC7rCHZwZ8hTpd6d01dF/1GvFPo8UhXE=
 github.com/cilium/controller-tools v0.8.0-1 h1:D5xhwSUZZceaKAacHOyfcpUMgLbs2TGeJEijNHlAQlc=
 github.com/cilium/controller-tools v0.8.0-1/go.mod h1:qE2DXhVOiEq5ijmINcFbqi9GZrrUjzB1TuJU0xa6eoY=
-github.com/cilium/ebpf v0.15.0 h1:7NxJhNiBT3NG8pZJ3c+yfrVdHY8ScgKD27sScgjLMMk=
-github.com/cilium/ebpf v0.15.0/go.mod h1:DHp1WyrLeiBh19Cf/tfiSMhqheEiK8fXFZ4No0P1Hso=
+github.com/cilium/ebpf v0.15.1-0.20240703142256-12aca1027ee8 h1:+AMtTVJtVC6OGInoYSGtTW9YTblQBi/sysXD4rtc3to=
+github.com/cilium/ebpf v0.15.1-0.20240703142256-12aca1027ee8/go.mod h1:L7u2Blt2jMM/vLAVgjxluxtBKlz3/GWjB0dMOEngfwE=
 github.com/cilium/little-vm-helper v0.0.18 h1:Sx3D9lQ6glUwWyF9b8I/sd/mo+2qobnpMGT1n6VlS04=
 github.com/cilium/little-vm-helper v0.0.18/go.mod h1:Cq9INShkRoeR4LC46dwHkfL3EZfHsN+e+xAsJKJ/wJM=
 github.com/cilium/lumberjack/v2 v2.3.0 h1:IhVJMvPpqDYmQzC0KDhAoy7KlaRsyOsZnT97Nsa3u0o=
@@ -322,8 +322,12 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA=
+github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
 github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629 h1:1dSBUfGlorLAua2CRx0zFN7kQsTpE2DQSmr7rrTNgY8=
 github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629/go.mod h1:mb5nS4uRANwOJSZj8rlCWAfAcGi72GGMIXx+xGOjA7M=
+github.com/jsimonetti/rtnetlink/v2 v2.0.1 h1:xda7qaHDSVOsADNouv7ukSuicKZO7GgVUCXxpaIEIlM=
+github.com/jsimonetti/rtnetlink/v2 v2.0.1/go.mod h1:7MoNYNbb3UaDHtF8udiJo/RH6VsTKP1pqKLUTVCvToE=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
@@ -372,6 +376,10 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g=
+github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw=
+github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U=
+github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA=
 github.com/mennanov/fieldmask-utils v1.1.2 h1:f5hd3hYeWdl+q2thiKYyZZmqTqn90uayWG03bca9U+E=
 github.com/mennanov/fieldmask-utils v1.1.2/go.mod h1:xRqd9Fjz/gFEDYCQw7pxGouxqLhSPrkOdx2yhEAXEls=
 github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=

pkg/bpf/detect.go

@@ -9,6 +9,7 @@ package bpf
 import (
 	"errors"
 	"fmt"
+	"path/filepath"
 	"sync"
 	"unsafe"
 
@@ -32,6 +33,7 @@ var (
 	buildid             Feature
 	modifyReturn        Feature
 	modifyReturnSyscall Feature
+	linkPin             Feature
 )
 
 func HasOverrideHelper() bool {
@@ -218,8 +220,49 @@ func HasProgramLargeSize() bool {
 	return features.HaveLargeInstructions() == nil
 }
 
+func detectLinkPin() (bool, error) {
+	prog, err := ebpf.NewProgram(&ebpf.ProgramSpec{
+		Name: "probe_bpf_kprobe",
+		Type: ebpf.Kprobe,
+		Instructions: asm.Instructions{
+			asm.Mov.Imm(asm.R0, 0),
+			asm.Return(),
+		},
+		License: "MIT",
+	})
+	if err != nil {
+		return false, err
+	}
+	defer prog.Close()
+
+	lnk, err := link.Kprobe("vprintk", prog, nil)
+	if err != nil {
+		return false, err
+	}
+	defer lnk.Close()
+
+	if err := lnk.Pin(filepath.Join(GetMapRoot(), "test-link")); err != nil {
+		return false, err
+	}
+	lnk.Unpin()
+	return true, nil
+}
+
+func HasLinkPin() bool {
+	linkPin.init.Do(func() {
+		var err error
+
+		linkPin.detected, err = detectLinkPin()
+		if err != nil {
+			logger.GetLogger().WithError(err).Error("detect link pin")
+		}
+	})
+	return linkPin.detected
+}
+
 func LogFeatures() string {
-	return fmt.Sprintf("override_return: %t, buildid: %t, kprobe_multi: %t, uprobe_multi %t, fmodret: %t, fmodret_syscall: %t, signal: %t, large: %t",
+	return fmt.Sprintf("override_return: %t, buildid: %t, kprobe_multi: %t, uprobe_multi %t, fmodret: %t, fmodret_syscall: %t, signal: %t, large: %t, link_pin: %t",
 		HasOverrideHelper(), HasBuildId(), HasKprobeMulti(), HasUprobeMulti(),
-		HasModifyReturn(), HasModifyReturnSyscall(), HasSignalHelper(), HasProgramLargeSize())
+		HasModifyReturn(), HasModifyReturnSyscall(), HasSignalHelper(), HasProgramLargeSize(),
+		HasLinkPin())
 }

pkg/option/config.go

@@ -34,6 +34,8 @@ type config struct {
 
 	GopsAddr string
 
+	// On start used to store bpf prefix for --bpf-dir option,
+	// then it's updated to cary the whole path
 	BpfDir string
 
 	LogOpts map[string]string
@@ -92,6 +94,8 @@ type config struct {
 
 	HealthServerAddress  string
 	HealthServerInterval int
+
+	KeepSensorsOnExit bool
 }
 
 var (

pkg/option/flags.go

@@ -103,6 +103,10 @@ const (
 
 	KeyHealthServerAddress = "health-server-address"
 	KeyHealthTimeInterval  = "health-server-interval"
+
+	KeyBpfDir = "bpf-dir"
+
+	KeyKeepSensorsOnExit = "keep-sensors-on-exit"
 )
 
 type UsernameMetadaCode int
@@ -218,6 +222,10 @@ func ReadAndSetFlags() error {
 	Config.CgroupRate = ParseCgroupRate(viper.GetString(KeyCgroupRate))
 	Config.HealthServerAddress = viper.GetString(KeyHealthServerAddress)
 	Config.HealthServerInterval = viper.GetInt(KeyHealthTimeInterval)
+
+	Config.BpfDir = viper.GetString(KeyBpfDir)
+
+	Config.KeepSensorsOnExit = viper.GetBool(KeyKeepSensorsOnExit)
 	return nil
 }
 
@@ -372,4 +380,8 @@ func AddFlags(flags *pflag.FlagSet) {
 
 	flags.String(KeyHealthServerAddress, ":6789", "Health server address (e.g. ':6789')(use '' to disabled it)")
 	flags.Int(KeyHealthTimeInterval, 10, "Health server interval in seconds")
+
+	flags.String(KeyBpfDir, defaults.DefaultMapPrefix, "Set tetragon bpf directory (default 'tetragon')")
+
+	flags.Bool(KeyKeepSensorsOnExit, false, "Do not unload sensors on exit")
 }