Added stable test. Renamed options. Fix docs.

Signed-off-by: Andrey Fedotov <[email protected]>

contrib/tester-progs/Makefile

@@ -22,7 +22,8 @@ PROGS = sigkill-tester \
 	drop-privileges \
 	getcpu \
 	direct-write-tester \
-	change-capabilities
+	change-capabilities \
+	user-stacktrace
 
 # For now enforcer-tester is compiled to 32-bit only on x86_64 as we want
 # to test 32-bit binaries and system calls compatibility layer.
@@ -88,6 +89,9 @@ lseek-pipe: FORCE
 getcpu: FORCE
 	go build -o getcpu ./go/getcpu
 
+user-stacktrace: FORCE
+	go build -o user-stacktrace ./go/user-stacktrace
+
 .PHONY: clean
 clean:
 	rm -f $(PROGS)

contrib/tester-progs/go/getcpu/getcpu.go

@@ -5,7 +5,6 @@ package main
 
 import (
 	"os"
-	"time"
 	"unsafe"
 
 	"golang.org/x/sys/unix"
@@ -19,6 +18,5 @@ func main() {
 		uintptr(unsafe.Pointer(&node)),
 		0,
 	)
-	time.Sleep(1 * time.Second)
 	os.Exit(int(err))
 }

contrib/tester-progs/go/user-stacktrace/user-stacktrace.go

@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Tetragon
+
+package main
+
+import (
+	"os"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+//nolint:all
+func main() {
+	// This test has endless loop for stable stack trace collection.
+	// It must be terminated from tests.
+	var cpu, node int
+	_, _, err := unix.Syscall(
+		unix.SYS_GETCPU,
+		uintptr(unsafe.Pointer(&cpu)),
+		uintptr(unsafe.Pointer(&node)),
+		0,
+	)
+	if err != 0 {
+		os.Exit(int(err))
+	}
+	for {
+	}
+}

docs/content/en/docs/concepts/tracing-policy/selectors.md

@@ -924,7 +924,7 @@ kprobes:
 
 {{< caution >}}
 By default Tetragon does not expose the linear addresses from kernel space or
-user space, you need to enable the flag `--expose-addresses` to get the addresses
+user space, you need to enable the flag `--expose-stack-addresses` to get the addresses
 along the rest.
 
 Note that the Tetragon agent is using its privilege to read the kernel symbols
@@ -987,6 +987,14 @@ beginning of the binary module. "module" is the absolute path of the binary file
 to which address belongs. "symbol" is the function symbol name. "symbol" may be missing
 if the binary file is stripped.
 
+{{< note >}}
+Information from `procfs (/proc/<pid>/maps)` is used to symbolize user
+stack trace addresses. Stack trace addresses extraction and symbolizing are async.
+It might happen that process is terminated and the `/proc/<pid>/maps` file will be
+not existed at user stack trace symbolization step. In such case user stack traces
+for very short living process might be not collected.
+{{< /note >}}
+
 This output can be enhanced in a more human friendly using the `tetra getevents
 -o compact` command. Indeed, by default, it will print the stack trace along
 the compact output of the event similarly to this:
@@ -1011,7 +1019,7 @@ The printing format for user stack trace is `"0x%x: %s (%s+0x%x)", address, symb
 
 {{< note >}}
 Compact output will display missing addresses as `0x0`, see the above note on
-`--expose-addresses` for more info.
+`--expose-stack-addresses` for more info.
 {{< /note >}}
 
 ### NoPost action

pkg/encoder/encoder.go

@@ -211,11 +211,13 @@ func HumanStackTrace(response *tetragon.GetEventsResponse, colorer *Colorer) str
 			}
 		}
 		if ev.ProcessKprobe.UserStackTrace != nil {
-			fmt.Fprintf(out, "User space:\n")
+			fmt.Fprintf(out, "User:\n")
 			for _, st := range ev.ProcessKprobe.UserStackTrace {
 				colorer.Green.Fprintf(out, "   0x%x:", st.Address)
-				colorer.Blue.Fprintf(out, " %s ", st.Symbol)
-				colorer.Yellow.Fprintf(out, "(%s+0x%x)\n", st.Module, st.Offset)
+				if st.Symbol != "" {
+					colorer.Blue.Fprintf(out, " %s", st.Symbol)
+				}
+				colorer.Yellow.Fprintf(out, " (%s+0x%x)\n", st.Module, st.Offset)
 			}
 		}
 	}

pkg/option/flags.go

@@ -90,7 +90,7 @@ const (
 	KeyEnablePodInfo          = "enable-pod-info"
 	KeyEnableTracingPolicyCRD = "enable-tracing-policy-crd"
 
-	KeyExposeAddresses = "expose-addresses"
+	KeyExposeStackAddresses = "expose-stack-addresses"
 
 	KeyGenerateDocs = "generate-docs"
 )
@@ -171,7 +171,7 @@ func ReadAndSetFlags() error {
 
 	Config.TracingPolicy = viper.GetString(KeyTracingPolicy)
 
-	Config.ExposeAddresses = viper.GetBool(KeyExposeAddresses)
+	Config.ExposeAddresses = viper.GetBool(KeyExposeStackAddresses)
 
 	return nil
 }
@@ -279,7 +279,7 @@ func AddFlags(flags *pflag.FlagSet) {
 	flags.Bool(KeyEnablePodInfo, false, "Enable PodInfo custom resource")
 	flags.Bool(KeyEnableTracingPolicyCRD, true, "Enable TracingPolicy and TracingPolicyNamespaced custom resources")
 
-	flags.Bool(KeyExposeAddresses, false, "Expose real linear addresses in events stack traces")
+	flags.Bool(KeyExposeStackAddresses, false, "Expose real linear addresses in events stack traces")
 
 	flags.Bool(KeyGenerateDocs, false, "Generate documentation in YAML format to stdout")
 }

pkg/procsyms/procsyms.go

@@ -116,17 +116,19 @@ func GetFnSymbol(pid int, addr uint64) (*FnSym, error) {
 			// Symbols are unordered by Value, so using linear scan
 			if symOffset <= sym.Offset && sym.Offset < (symOffset+s.Size) {
 				sym.Name = s.Name
-				key := struct {
-					module string
-					offset uint64
-				}{
-					module: sym.Module,
-					offset: sym.Offset,
-				}
-				cache.Add(key, sym.Name)
 				break
 			}
 		}
+
+		// Store sym in cache, no matter was it found or not.
+		key := struct {
+			module string
+			offset uint64
+		}{
+			module: sym.Module,
+			offset: sym.Offset,
+		}
+		cache.Add(key, sym.Name)
 	}
 	return &sym, nil
 }

pkg/sensors/tracing/kprobe_test.go

@@ -6148,7 +6148,7 @@ func TestKprobeUserStackTrace(t *testing.T) {
 
 	ctx, cancel := context.WithTimeout(context.Background(), tus.Conf().CmdWaitTime)
 	defer cancel()
-	testGetCpu := testutils.RepoRootPath("contrib/tester-progs/getcpu")
+	testUserStacktrace := testutils.RepoRootPath("contrib/tester-progs/user-stacktrace")
 	tracingPolicy := `apiVersion: cilium.io/v1alpha1
 kind: TracingPolicy
 metadata:
@@ -6160,7 +6160,7 @@ spec:
     - matchBinaries:
       - operator: "In"
         values:
-        - "` + testGetCpu + `"
+        - "` + testUserStacktrace + `"
       matchActions:
       - action: Post
         userStackTrace: true`
@@ -6173,27 +6173,32 @@ spec:
 	}
 	observertesthelper.LoopEvents(ctx, t, &doneWG, &readyWG, obs)
 	readyWG.Wait()
+	test_cmd := exec.Command(testUserStacktrace)
 
-	if err := exec.Command(testGetCpu).Run(); err != nil {
-		t.Fatalf("failed to run %s: %s", testGetCpu, err)
+	if err := test_cmd.Start(); err != nil {
+		t.Fatalf("failed to run %s: %s", testUserStacktrace, err)
 	}
 
 	stackTraceChecker := ec.NewProcessKprobeChecker("user-stack-trace").
-		WithProcess(ec.NewProcessChecker().WithBinary(sm.Full(testGetCpu))).
+		WithProcess(ec.NewProcessChecker().WithBinary(sm.Full(testUserStacktrace))).
 		WithUserStackTrace(ec.NewStackTraceEntryListMatcher().WithValues(
 			ec.NewStackTraceEntryChecker().WithSymbol(sm.Suffix(("main.main"))),
-			// syscall user-nix /home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/getcpu __x64_sys_getcpu
-			// User space:
-			// 0x0: runtime/internal/syscall.Syscall6 (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/getcpu+0x2aee)
-			// 0x0: syscall.Syscall (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/getcpu+0x635e6)
-			// 0x0: syscall.Syscall.abi0 (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/getcpu+0x6374e)
-			// 0x0: main.main (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/getcpu+0x652e5)
-			// 0x0: runtime.main (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/getcpu+0x3313d)
-			// 0x0: runtime.goexit.abi0 (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/getcpu+0x5e901)
+			// syscall user-nix /home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/user-stacktrace __x64_sys_getcpu
+			// User:
+			//   0x0: runtime/internal/syscall.Syscall6 (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/user-stacktrace+0x2aee)
+			//   0x0: syscall.Syscall (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/user-stacktrace+0x63346)
+			//   0x0: syscall.Syscall.abi0 (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/user-stacktrace+0x634ae)
+			//   0x0: main.main (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/user-stacktrace+0x6503e)
+			//   0x0: runtime.main (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/user-stacktrace+0x3313d)
+			//   0x0: runtime.goexit.abi0 (/home/user/go/src/github.com/cilium/tetragon/contrib/tester-progs/user-stacktrace+0x5e661)
 		))
 
 	checker := ec.NewUnorderedEventChecker(stackTraceChecker)
 	err = jsonchecker.JsonTestCheck(t, checker)
+
+	// Kill test because of endless loop in the test for stable stack trace extraction
+	test_cmd.Process.Kill()
+
 	assert.NoError(t, err)
 }