backport/v0.10: bugtool gops pprofheap

.github/workflows/packages-e2e-tests.yaml

@@ -0,0 +1,90 @@
+name: Packages e2e Tests
+
+on:
+  pull_request:
+    paths-ignore:
+    - "**.md"
+    - 'docs/**'
+
+jobs:
+  standalone-tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # We use the native arch build
+          - os: ubuntu-22.04
+            arch: amd64
+            match_arch: x86-64
+
+    steps:
+      # https://github.com/docker/setup-buildx-action
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
+
+      - name: Checkout Source Code
+        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+          submodules: true
+
+      - name: Getting version tag
+        id: tag
+        run: echo "tag=$(make version)" >> $GITHUB_OUTPUT
+
+      # Logging into Docker Hub makes the job less flaky, but it relies on repo secrets so
+      # does not work for external contributors. Let's compromise and do this for internal
+      # PRs at least (for now). We can always improve the situation later.
+      - name: Login to Docker Hub
+        if: ${{ github.event.pull_request.head.repo.full_name == 'cilium/tetragon' }}
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME_CI }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD_CI }}
+
+      - name: Generate Tetragon Tarball
+        id: tetragon-tarball
+        run: make tarball
+
+      - name: Configure gops
+        run: |
+          sudo mkdir -p /etc/tetragon/tetragon.conf.d/
+          echo "localhost:8118" | sudo tee /etc/tetragon/tetragon.conf.d/gops-address
+
+      - name: Install Tetragon Tarball
+        run: |
+          tar zxvf tetragon-${{ steps.tag.outputs.tag }}-${{ matrix.arch }}.tar.gz
+          sudo ./tetragon-${{ steps.tag.outputs.tag }}-${{ matrix.arch }}/install.sh
+        working-directory: ./build/${{ matrix.arch }}/linux-tarball/
+
+      - name: Wait for Tetragon service
+        uses: nick-fields/retry@14672906e672a08bd6eeb15720e9ed3ce869cdd4 # v2
+        with:
+          timeout_seconds: 30
+          max_attempts: 5
+          retry_wait_seconds: 5
+          retry_on: error
+          command: |
+            # Ensure that default native builds work
+            file /usr/local/bin/tetragon | grep ${{ matrix.match_arch }} -
+            sudo systemctl is-active tetragon
+            sudo tetra status
+
+      - name: Check Tetragon startup logs
+        run: sudo journalctl -b -u tetragon --no-pager
+
+      - name: Test Tetragon
+        run: |
+          sudo tetra status
+          sudo grep "tetra" /var/log/tetragon/tetragon.log
+          sudo tetra tracingpolicy list
+          sudo tetra bugtool
+          test $(stat -c %a /var/run/tetragon/tetragon.sock) -eq "660"
+          sudo tetra bugtool 2>&1 | grep "Successfully dumped gops pprof-heap" -
+
+      - name: Uninstall Tetragon Tarball
+        run: |
+          sudo ./tetragon-${{ steps.tag.outputs.tag }}-${{ matrix.arch }}/uninstall.sh
+        working-directory: ./build/${{ matrix.arch }}/linux-tarball/

Dockerfile.tarball

@@ -21,6 +21,7 @@ COPY --from=tetragon-release /usr/bin/tetra /usr/local/bin/
 
 # Helper binaries
 COPY --from=tetragon-release /usr/bin/bpftool /usr/local/lib/tetragon/
+COPY --from=tetragon-release /usr/bin/gops /usr/local/lib/tetragon/
 
 # BPF files
 COPY --from=tetragon-release /var/lib/tetragon /usr/local/lib/tetragon/bpf

cmd/tetra/bugtool/bugtool.go

@@ -23,19 +23,21 @@ import (
 var (
 	outFile string
 	bpfTool string
+	gops    string
 )
 
 func New() *cobra.Command {
 	bugtoolCmd := &cobra.Command{
 		Use:   "bugtool",
 		Short: "Produce a tar archive with debug information",
 		Run: func(cmd *cobra.Command, args []string) {
-			bugtool.Bugtool(outFile, bpfTool)
+			bugtool.Bugtool(outFile, bpfTool, gops)
 		},
 	}
 
 	flags := bugtoolCmd.Flags()
 	flags.StringVarP(&outFile, "out", "o", "tetragon-bugtool.tar.gz", "Output filename")
 	flags.StringVar(&bpfTool, "bpftool", "", "Path to bpftool binary")
+	flags.StringVar(&gops, "gops", "", "Path to gops binary")
 	return bugtoolCmd
 }

pkg/bugtool/bugtool.go

@@ -14,6 +14,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"os"
 	"os/exec"
@@ -26,6 +27,7 @@ import (
 	"github.com/cilium/tetragon/pkg/defaults"
 	"github.com/cilium/tetragon/pkg/logger"
 	"github.com/cilium/tetragon/pkg/policyfilter"
+	gopssignal "github.com/google/gops/signal"
 	"go.uber.org/multierr"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
@@ -44,6 +46,7 @@ type InitInfo struct {
 	GopsAddr    string `json:"gops_address"`
 	MapDir      string `json:"map_dir"`
 	BpfToolPath string `json:"bpftool_path"`
+	GopsPath    string `json:"gops_path"`
 }
 
 // LoadInitInfo returns the InitInfo by reading the info file from its default location
@@ -83,6 +86,14 @@ func doSaveInitInfo(fname string, info *InitInfo) error {
 		logger.GetLogger().WithField("bpftool", info.BpfToolPath).Info("Successfully detected bpftool path")
 	}
 
+	gops, err := exec.LookPath("gops")
+	if err != nil {
+		logger.GetLogger().Warn("failed to locate gops binary, on bugtool debugging ensure you have gops installed")
+	} else {
+		info.GopsPath = gops
+		logger.GetLogger().WithField("gops", info.GopsPath).Info("Successfully detected gops path")
+	}
+
 	// Create DefaultRunDir if it does not already exist
 	if err := os.MkdirAll(defaults.DefaultRunDir, 0755); err != nil {
 		logger.GetLogger().WithField("infoFile", fname).Warn("failed to directory exists")
@@ -182,7 +193,7 @@ func (s *bugtoolInfo) tarAddFile(tarWriter *tar.Writer, fnameSrc string, fnameDs
 }
 
 // Bugtool gathers information and writes it as a tar archive in the given filename
-func Bugtool(outFname string, bpftool string) error {
+func Bugtool(outFname string, bpftool string, gops string) error {
 	info, err := LoadInitInfo()
 	if err != nil {
 		return err
@@ -192,6 +203,10 @@ func Bugtool(outFname string, bpftool string) error {
 		info.BpfToolPath = bpftool
 	}
 
+	if gops != "" {
+		info.GopsPath = gops
+	}
+
 	return doBugtool(info, outFname)
 }
 
@@ -488,21 +503,69 @@ func (s *bugtoolInfo) addBpftoolInfo(tarWriter *tar.Writer) {
 
 	_, err := os.Stat(s.info.BpfToolPath)
 	if err != nil {
-		s.multiLog.WithError(err).Warn("Failed to locate bpftool, please install it.")
+		s.multiLog.WithError(err).Warn("Failed to locate bpftool. Please install it or specify its path, see 'bugtool --help'")
 		return
 	}
 	s.execCmd(tarWriter, "bpftool-maps.json", s.info.BpfToolPath, "map", "show", "-j")
 	s.execCmd(tarWriter, "bpftool-progs.json", s.info.BpfToolPath, "prog", "show", "-j")
 	s.execCmd(tarWriter, "bpftool-cgroups.json", s.info.BpfToolPath, "cgroup", "tree", "-j")
 }
 
+func (s *bugtoolInfo) getPProf(tarWriter *tar.Writer, file string) error {
+	if s.info.GopsAddr == "" {
+		s.multiLog.Info("Skipping gops dump info as daemon is running without gops, use --gops-address to enable gops")
+		return nil
+	}
+
+	s.multiLog.WithField("gops-address", s.info.GopsAddr).Info("Contacting gops server for pprof dump")
+
+	conn, err := net.Dial("tcp", s.info.GopsAddr)
+	if err != nil {
+		s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to contact gops server")
+		return err
+	}
+
+	buf := []byte{gopssignal.HeapProfile}
+	if _, err := conn.Write(buf); err != nil {
+		s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to send gops pprof-heap command")
+		return err
+	}
+
+	buff := new(bytes.Buffer)
+	if _, err = buff.ReadFrom(conn); err != nil {
+		s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed reading gops pprof-heap response")
+	}
+	return s.tarAddBuff(tarWriter, file, buff)
+}
+
 func (s *bugtoolInfo) addGopsInfo(tarWriter *tar.Writer) {
 	if s.info.GopsAddr == "" {
+		s.multiLog.Info("Skipping gops dump info as daemon is running without gops, use --gops-address to enable gops")
 		return
 	}
-	s.execCmd(tarWriter, "gops.stack", "gops", "stack", s.info.GopsAddr)
-	s.execCmd(tarWriter, "gpos.stats", "gops", "stats", s.info.GopsAddr)
-	s.execCmd(tarWriter, "gops.memstats", "gops", "memstats", s.info.GopsAddr)
+
+	if s.info.GopsPath == "" {
+		s.multiLog.WithField("gops-address", s.info.GopsAddr).Warn("Failed to locate gops. Please install it or specify its path, see 'bugtool --help'")
+		return
+	}
+
+	_, err := os.Stat(s.info.GopsPath)
+	if err != nil {
+		s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to locate gops, please install it")
+		return
+	}
+
+	s.multiLog.WithField("gops-address", s.info.GopsAddr).WithField("gops-path", s.info.GopsPath).Info("Dumping gops information")
+
+	s.execCmd(tarWriter, "gops.stack", s.info.GopsPath, "stack", s.info.GopsAddr)
+	s.execCmd(tarWriter, "gops.stats", s.info.GopsPath, "stats", s.info.GopsAddr)
+	s.execCmd(tarWriter, "gops.memstats", s.info.GopsPath, "memstats", s.info.GopsAddr)
+	err = s.getPProf(tarWriter, "gops.pprof-heap")
+	if err != nil {
+		s.multiLog.WithField("gops-address", s.info.GopsAddr).WithField("gops-path", s.info.GopsPath).WithError(err).Warn("Failed to dump gops pprof-heap")
+	} else {
+		s.multiLog.WithField("gops-address", s.info.GopsAddr).WithField("gops-path", s.info.GopsPath).Info("Successfully dumped gops pprof-heap")
+	}
 }
 
 func (s *bugtoolInfo) dumpPolicyFilterMap(tarWriter *tar.Writer) error {

pkg/observer/observer_test_helper.go

@@ -129,7 +129,7 @@ func withNotestfail(notestfail bool) TestOption {
 func testDone(t *testing.T, obs *Observer) {
 	if t.Failed() {
 		bugtoolFname := "/tmp/tetragon-bugtool.tar.gz"
-		if err := bugtool.Bugtool(bugtoolFname, ""); err == nil {
+		if err := bugtool.Bugtool(bugtoolFname, "", ""); err == nil {
 			logger.GetLogger().WithField("test", t.Name()).
 				WithField("file", bugtoolFname).Info("Dumped bugtool info")
 		} else {