From 59b0e354143ee44f7f29732b7f0251c1de5b3e95 Mon Sep 17 00:00:00 2001 From: Djalal Harouni Date: Mon, 22 Jan 2024 14:53:33 +0100 Subject: [PATCH 1/4] bugtool: save gops path if any and allow to change it This adds support to get or set the gops path. Signed-off-by: Djalal Harouni --- cmd/tetra/bugtool/bugtool.go | 4 +- pkg/bugtool/bugtool.go | 38 ++++++++++++++++--- .../observer_test_helper.go | 2 +- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/cmd/tetra/bugtool/bugtool.go b/cmd/tetra/bugtool/bugtool.go index 45019608148..ffd15b8dd99 100644 --- a/cmd/tetra/bugtool/bugtool.go +++ b/cmd/tetra/bugtool/bugtool.go @@ -12,6 +12,7 @@ import ( var ( outFile string bpfTool string + gops string ) func New() *cobra.Command { @@ -19,12 +20,13 @@ func New() *cobra.Command { Use: "bugtool", Short: "Produce a tar archive with debug information", Run: func(cmd *cobra.Command, args []string) { - bugtool.Bugtool(outFile, bpfTool) + bugtool.Bugtool(outFile, bpfTool, gops) }, } flags := bugtoolCmd.Flags() flags.StringVarP(&outFile, "out", "o", "tetragon-bugtool.tar.gz", "Output filename") flags.StringVar(&bpfTool, "bpftool", "", "Path to bpftool binary") + flags.StringVar(&gops, "gops", "", "Path to gops binary") return bugtoolCmd } diff --git a/pkg/bugtool/bugtool.go b/pkg/bugtool/bugtool.go index 389a7dcd5c7..882cdb4e369 100644 --- a/pkg/bugtool/bugtool.go +++ b/pkg/bugtool/bugtool.go @@ -44,6 +44,7 @@ type InitInfo struct { GopsAddr string `json:"gops_address"` MapDir string `json:"map_dir"` BpfToolPath string `json:"bpftool_path"` + GopsPath string `json:"gops_path"` } // LoadInitInfo returns the InitInfo by reading the info file from its default location @@ -83,6 +84,14 @@ func doSaveInitInfo(fname string, info *InitInfo) error { logger.GetLogger().WithField("bpftool", info.BpfToolPath).Info("Successfully detected bpftool path") } + gops, err := exec.LookPath("gops") + if err != nil { + logger.GetLogger().Warn("failed to locate gops binary, on bugtool debugging ensure you have gops installed") + } else { + info.GopsPath = gops + logger.GetLogger().WithField("gops", info.GopsPath).Info("Successfully detected gops path") + } + // Create DefaultRunDir if it does not already exist if err := os.MkdirAll(defaults.DefaultRunDir, 0755); err != nil { logger.GetLogger().WithField("infoFile", fname).Warn("failed to directory exists") @@ -182,7 +191,7 @@ func (s *bugtoolInfo) tarAddFile(tarWriter *tar.Writer, fnameSrc string, fnameDs } // Bugtool gathers information and writes it as a tar archive in the given filename -func Bugtool(outFname string, bpftool string) error { +func Bugtool(outFname string, bpftool string, gops string) error { info, err := LoadInitInfo() if err != nil { return err @@ -192,6 +201,10 @@ func Bugtool(outFname string, bpftool string) error { info.BpfToolPath = bpftool } + if gops != "" { + info.GopsPath = gops + } + return doBugtool(info, outFname) } @@ -488,7 +501,7 @@ func (s *bugtoolInfo) addBpftoolInfo(tarWriter *tar.Writer) { _, err := os.Stat(s.info.BpfToolPath) if err != nil { - s.multiLog.WithError(err).Warn("Failed to locate bpftool, please install it.") + s.multiLog.WithError(err).Warn("Failed to locate bpftool. Please install it or specify its path, see 'bugtool --help'") return } s.execCmd(tarWriter, "bpftool-maps.json", s.info.BpfToolPath, "map", "show", "-j") @@ -498,11 +511,26 @@ func (s *bugtoolInfo) addBpftoolInfo(tarWriter *tar.Writer) { func (s *bugtoolInfo) addGopsInfo(tarWriter *tar.Writer) { if s.info.GopsAddr == "" { + s.multiLog.Info("Skipping gops dump info as daemon is running without gops, use --gops-address to enable gops") return } - s.execCmd(tarWriter, "gops.stack", "gops", "stack", s.info.GopsAddr) - s.execCmd(tarWriter, "gpos.stats", "gops", "stats", s.info.GopsAddr) - s.execCmd(tarWriter, "gops.memstats", "gops", "memstats", s.info.GopsAddr) + + if s.info.GopsPath == "" { + s.multiLog.WithField("gops-address", s.info.GopsAddr).Warn("Failed to locate gops. Please install it or specify its path, see 'bugtool --help'") + return + } + + _, err := os.Stat(s.info.GopsPath) + if err != nil { + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to locate gops, please install it.") + return + } + + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithField("gops-path", s.info.GopsPath).Info("Dumping gops information") + + s.execCmd(tarWriter, "gops.stack", s.info.GopsPath, "stack", s.info.GopsAddr) + s.execCmd(tarWriter, "gops.stats", s.info.GopsPath, "stats", s.info.GopsAddr) + s.execCmd(tarWriter, "gops.memstats", s.info.GopsPath, "memstats", s.info.GopsAddr) } func (s *bugtoolInfo) dumpPolicyFilterMap(tarWriter *tar.Writer) error { diff --git a/pkg/observer/observertesthelper/observer_test_helper.go b/pkg/observer/observertesthelper/observer_test_helper.go index b98428db0a5..1407d5ba80c 100644 --- a/pkg/observer/observertesthelper/observer_test_helper.go +++ b/pkg/observer/observertesthelper/observer_test_helper.go @@ -125,7 +125,7 @@ func WithLib(lib string) TestOption { func testDone(tb testing.TB, obs *observer.Observer) { if tb.Failed() { bugtoolFname := "/tmp/tetragon-bugtool.tar.gz" - if err := bugtool.Bugtool(bugtoolFname, ""); err == nil { + if err := bugtool.Bugtool(bugtoolFname, "", ""); err == nil { logger.GetLogger().WithField("test", tb.Name()). WithField("file", bugtoolFname).Info("Dumped bugtool info") } else { From 45a2a2abe39db86422b91dbd1e506d3224bf41be Mon Sep 17 00:00:00 2001 From: Djalal Harouni Date: Wed, 24 Jan 2024 16:34:30 +0100 Subject: [PATCH 2/4] pprof: get pprof-heap profile from gops We use gops to get the pprof heap profile instead of http, as it seems that gops does not export an HTTP endpoint, and by default we do not start the pprof profiling server. The --pprof-addr is marked hidden and requiring users to listen on another port could be an overkill. Signed-off-by: Djalal Harouni --- pkg/bugtool/bugtool.go | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/pkg/bugtool/bugtool.go b/pkg/bugtool/bugtool.go index 882cdb4e369..759e0a142d3 100644 --- a/pkg/bugtool/bugtool.go +++ b/pkg/bugtool/bugtool.go @@ -14,6 +14,7 @@ import ( "errors" "fmt" "io" + "net" "net/http" "os" "os/exec" @@ -26,6 +27,7 @@ import ( "github.com/cilium/tetragon/pkg/defaults" "github.com/cilium/tetragon/pkg/logger" "github.com/cilium/tetragon/pkg/policyfilter" + gopssignal "github.com/google/gops/signal" "go.uber.org/multierr" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -509,6 +511,33 @@ func (s *bugtoolInfo) addBpftoolInfo(tarWriter *tar.Writer) { s.execCmd(tarWriter, "bpftool-cgroups.json", s.info.BpfToolPath, "cgroup", "tree", "-j") } +func (s *bugtoolInfo) getPProf(tarWriter *tar.Writer, file string) error { + if s.info.GopsAddr == "" { + s.multiLog.Info("Skipping gops dump info as daemon is running without gops, use --gops-address to enable gops") + return nil + } + + s.multiLog.WithField("gops-address", s.info.GopsAddr).Info("Contacting gops server for pprof dump") + + conn, err := net.Dial("tcp", s.info.GopsAddr) + if err != nil { + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to contact gops server") + return err + } + + buf := []byte{gopssignal.HeapProfile} + if _, err := conn.Write(buf); err != nil { + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to send gops pprof-heap command") + return err + } + + buff := new(bytes.Buffer) + if _, err = buff.ReadFrom(conn); err != nil { + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed reading gops pprof-heap response") + } + return s.tarAddBuff(tarWriter, file, buff) +} + func (s *bugtoolInfo) addGopsInfo(tarWriter *tar.Writer) { if s.info.GopsAddr == "" { s.multiLog.Info("Skipping gops dump info as daemon is running without gops, use --gops-address to enable gops") @@ -522,7 +551,7 @@ func (s *bugtoolInfo) addGopsInfo(tarWriter *tar.Writer) { _, err := os.Stat(s.info.GopsPath) if err != nil { - s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to locate gops, please install it.") + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to locate gops, please install it") return } @@ -531,6 +560,12 @@ func (s *bugtoolInfo) addGopsInfo(tarWriter *tar.Writer) { s.execCmd(tarWriter, "gops.stack", s.info.GopsPath, "stack", s.info.GopsAddr) s.execCmd(tarWriter, "gops.stats", s.info.GopsPath, "stats", s.info.GopsAddr) s.execCmd(tarWriter, "gops.memstats", s.info.GopsPath, "memstats", s.info.GopsAddr) + err = s.getPProf(tarWriter, "gops.pprof-heap") + if err != nil { + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithField("gops-path", s.info.GopsPath).WithError(err).Warn("Failed to dump gops pprof-heap") + } else { + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithField("gops-path", s.info.GopsPath).Info("Successfully dumped gops pprof-heap") + } } func (s *bugtoolInfo) dumpPolicyFilterMap(tarWriter *tar.Writer) error { From dd016198f0694fb9ed24152391c1783e80bc9ba3 Mon Sep 17 00:00:00 2001 From: Djalal Harouni Date: Mon, 22 Jan 2024 20:25:36 +0100 Subject: [PATCH 3/4] tarball: copy gops into tarball Signed-off-by: Djalal Harouni --- Dockerfile.tarball | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile.tarball b/Dockerfile.tarball index 6339ff07356..114031114d7 100644 --- a/Dockerfile.tarball +++ b/Dockerfile.tarball @@ -21,6 +21,7 @@ COPY --from=tetragon-release /usr/bin/tetra /usr/local/bin/ # Helper binaries COPY --from=tetragon-release /usr/bin/bpftool /usr/local/lib/tetragon/ +COPY --from=tetragon-release /usr/bin/gops /usr/local/lib/tetragon/ # BPF files COPY --from=tetragon-release /var/lib/tetragon /usr/local/lib/tetragon/bpf From bb0139cfa00ed181846bcb7ff22771ac3a60d53d Mon Sep 17 00:00:00 2001 From: Djalal Harouni Date: Mon, 22 Jan 2024 20:23:42 +0100 Subject: [PATCH 4/4] ci:tarball: ensure that bugtool gops heap works Signed-off-by: Djalal Harouni --- .github/workflows/packages-e2e-tests.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/packages-e2e-tests.yaml b/.github/workflows/packages-e2e-tests.yaml index 7fa01f18cc6..2dfcc2bc35a 100644 --- a/.github/workflows/packages-e2e-tests.yaml +++ b/.github/workflows/packages-e2e-tests.yaml @@ -142,6 +142,7 @@ jobs: sudo mkdir -p /opt/tetragon.tp.d/ sudo cp examples/tracingpolicy/bpf.yaml /opt/tetragon.tp.d/bpf.yaml echo "/opt/tetragon.tp.d/" | sudo tee /etc/tetragon/tetragon.conf.d/tracing-policy-dir + echo "localhost:8118" | sudo tee /etc/tetragon/tetragon.conf.d/gops-address sudo systemctl start tetragon - name: Test Tetragon with a different tracing-policy-dir @@ -156,7 +157,7 @@ jobs: sudo tetra status sudo grep "tetra" /var/log/tetragon/tetragon.log sudo tetra tracingpolicy list | grep bpf - - sudo tetra bugtool + sudo tetra bugtool 2>&1 | grep "Successfully dumped gops pprof-heap" - - name: Uninstall Tetragon Tarball run: |