From 5fc667a3dca5c6e8c9469449c8f961a37e22ed1f Mon Sep 17 00:00:00 2001 From: Djalal Harouni Date: Thu, 1 Feb 2024 22:31:36 +0100 Subject: [PATCH 1/4] bugtool: save gops path if any and allow to change it [ Upstream master 36cae019ebc5b10d ] Patch fixed to resolve conflicts. This adds support to get or set the gops path. Signed-off-by: Djalal Harouni --- cmd/tetra/bugtool/bugtool.go | 4 ++- pkg/bugtool/bugtool.go | 38 ++++++++++++++++++++++++---- pkg/observer/observer_test_helper.go | 2 +- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/cmd/tetra/bugtool/bugtool.go b/cmd/tetra/bugtool/bugtool.go index 93e7ccd73a8..e7d4eb41b00 100644 --- a/cmd/tetra/bugtool/bugtool.go +++ b/cmd/tetra/bugtool/bugtool.go @@ -23,6 +23,7 @@ import ( var ( outFile string bpfTool string + gops string ) func New() *cobra.Command { @@ -30,12 +31,13 @@ func New() *cobra.Command { Use: "bugtool", Short: "Produce a tar archive with debug information", Run: func(cmd *cobra.Command, args []string) { - bugtool.Bugtool(outFile, bpfTool) + bugtool.Bugtool(outFile, bpfTool, gops) }, } flags := bugtoolCmd.Flags() flags.StringVarP(&outFile, "out", "o", "tetragon-bugtool.tar.gz", "Output filename") flags.StringVar(&bpfTool, "bpftool", "", "Path to bpftool binary") + flags.StringVar(&gops, "gops", "", "Path to gops binary") return bugtoolCmd } diff --git a/pkg/bugtool/bugtool.go b/pkg/bugtool/bugtool.go index 389a7dcd5c7..882cdb4e369 100644 --- a/pkg/bugtool/bugtool.go +++ b/pkg/bugtool/bugtool.go @@ -44,6 +44,7 @@ type InitInfo struct { GopsAddr string `json:"gops_address"` MapDir string `json:"map_dir"` BpfToolPath string `json:"bpftool_path"` + GopsPath string `json:"gops_path"` } // LoadInitInfo returns the InitInfo by reading the info file from its default location @@ -83,6 +84,14 @@ func doSaveInitInfo(fname string, info *InitInfo) error { logger.GetLogger().WithField("bpftool", info.BpfToolPath).Info("Successfully detected bpftool path") } + gops, err := exec.LookPath("gops") + if err != nil { + logger.GetLogger().Warn("failed to locate gops binary, on bugtool debugging ensure you have gops installed") + } else { + info.GopsPath = gops + logger.GetLogger().WithField("gops", info.GopsPath).Info("Successfully detected gops path") + } + // Create DefaultRunDir if it does not already exist if err := os.MkdirAll(defaults.DefaultRunDir, 0755); err != nil { logger.GetLogger().WithField("infoFile", fname).Warn("failed to directory exists") @@ -182,7 +191,7 @@ func (s *bugtoolInfo) tarAddFile(tarWriter *tar.Writer, fnameSrc string, fnameDs } // Bugtool gathers information and writes it as a tar archive in the given filename -func Bugtool(outFname string, bpftool string) error { +func Bugtool(outFname string, bpftool string, gops string) error { info, err := LoadInitInfo() if err != nil { return err @@ -192,6 +201,10 @@ func Bugtool(outFname string, bpftool string) error { info.BpfToolPath = bpftool } + if gops != "" { + info.GopsPath = gops + } + return doBugtool(info, outFname) } @@ -488,7 +501,7 @@ func (s *bugtoolInfo) addBpftoolInfo(tarWriter *tar.Writer) { _, err := os.Stat(s.info.BpfToolPath) if err != nil { - s.multiLog.WithError(err).Warn("Failed to locate bpftool, please install it.") + s.multiLog.WithError(err).Warn("Failed to locate bpftool. Please install it or specify its path, see 'bugtool --help'") return } s.execCmd(tarWriter, "bpftool-maps.json", s.info.BpfToolPath, "map", "show", "-j") @@ -498,11 +511,26 @@ func (s *bugtoolInfo) addBpftoolInfo(tarWriter *tar.Writer) { func (s *bugtoolInfo) addGopsInfo(tarWriter *tar.Writer) { if s.info.GopsAddr == "" { + s.multiLog.Info("Skipping gops dump info as daemon is running without gops, use --gops-address to enable gops") return } - s.execCmd(tarWriter, "gops.stack", "gops", "stack", s.info.GopsAddr) - s.execCmd(tarWriter, "gpos.stats", "gops", "stats", s.info.GopsAddr) - s.execCmd(tarWriter, "gops.memstats", "gops", "memstats", s.info.GopsAddr) + + if s.info.GopsPath == "" { + s.multiLog.WithField("gops-address", s.info.GopsAddr).Warn("Failed to locate gops. Please install it or specify its path, see 'bugtool --help'") + return + } + + _, err := os.Stat(s.info.GopsPath) + if err != nil { + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to locate gops, please install it.") + return + } + + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithField("gops-path", s.info.GopsPath).Info("Dumping gops information") + + s.execCmd(tarWriter, "gops.stack", s.info.GopsPath, "stack", s.info.GopsAddr) + s.execCmd(tarWriter, "gops.stats", s.info.GopsPath, "stats", s.info.GopsAddr) + s.execCmd(tarWriter, "gops.memstats", s.info.GopsPath, "memstats", s.info.GopsAddr) } func (s *bugtoolInfo) dumpPolicyFilterMap(tarWriter *tar.Writer) error { diff --git a/pkg/observer/observer_test_helper.go b/pkg/observer/observer_test_helper.go index 2aab6bf84ea..ef9bca684d1 100644 --- a/pkg/observer/observer_test_helper.go +++ b/pkg/observer/observer_test_helper.go @@ -129,7 +129,7 @@ func withNotestfail(notestfail bool) TestOption { func testDone(t *testing.T, obs *Observer) { if t.Failed() { bugtoolFname := "/tmp/tetragon-bugtool.tar.gz" - if err := bugtool.Bugtool(bugtoolFname, ""); err == nil { + if err := bugtool.Bugtool(bugtoolFname, "", ""); err == nil { logger.GetLogger().WithField("test", t.Name()). WithField("file", bugtoolFname).Info("Dumped bugtool info") } else { From a0290595b967d66248ec7ba5fa9f577f47827b4b Mon Sep 17 00:00:00 2001 From: Djalal Harouni Date: Thu, 1 Feb 2024 22:32:52 +0100 Subject: [PATCH 2/4] [PATCH 2/4] pprof: get pprof-heap profile from gops [Upstream main bae3b7746f36a563] We use gops to get the pprof heap profile instead of http, as it seems that gops does not export an HTTP endpoint, and by default we do not start the pprof profiling server. The --pprof-addr is marked hidden and requiring users to listen on another port could be an overkill. Signed-off-by: Djalal Harouni --- pkg/bugtool/bugtool.go | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/pkg/bugtool/bugtool.go b/pkg/bugtool/bugtool.go index 882cdb4e369..759e0a142d3 100644 --- a/pkg/bugtool/bugtool.go +++ b/pkg/bugtool/bugtool.go @@ -14,6 +14,7 @@ import ( "errors" "fmt" "io" + "net" "net/http" "os" "os/exec" @@ -26,6 +27,7 @@ import ( "github.com/cilium/tetragon/pkg/defaults" "github.com/cilium/tetragon/pkg/logger" "github.com/cilium/tetragon/pkg/policyfilter" + gopssignal "github.com/google/gops/signal" "go.uber.org/multierr" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -509,6 +511,33 @@ func (s *bugtoolInfo) addBpftoolInfo(tarWriter *tar.Writer) { s.execCmd(tarWriter, "bpftool-cgroups.json", s.info.BpfToolPath, "cgroup", "tree", "-j") } +func (s *bugtoolInfo) getPProf(tarWriter *tar.Writer, file string) error { + if s.info.GopsAddr == "" { + s.multiLog.Info("Skipping gops dump info as daemon is running without gops, use --gops-address to enable gops") + return nil + } + + s.multiLog.WithField("gops-address", s.info.GopsAddr).Info("Contacting gops server for pprof dump") + + conn, err := net.Dial("tcp", s.info.GopsAddr) + if err != nil { + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to contact gops server") + return err + } + + buf := []byte{gopssignal.HeapProfile} + if _, err := conn.Write(buf); err != nil { + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to send gops pprof-heap command") + return err + } + + buff := new(bytes.Buffer) + if _, err = buff.ReadFrom(conn); err != nil { + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed reading gops pprof-heap response") + } + return s.tarAddBuff(tarWriter, file, buff) +} + func (s *bugtoolInfo) addGopsInfo(tarWriter *tar.Writer) { if s.info.GopsAddr == "" { s.multiLog.Info("Skipping gops dump info as daemon is running without gops, use --gops-address to enable gops") @@ -522,7 +551,7 @@ func (s *bugtoolInfo) addGopsInfo(tarWriter *tar.Writer) { _, err := os.Stat(s.info.GopsPath) if err != nil { - s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to locate gops, please install it.") + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to locate gops, please install it") return } @@ -531,6 +560,12 @@ func (s *bugtoolInfo) addGopsInfo(tarWriter *tar.Writer) { s.execCmd(tarWriter, "gops.stack", s.info.GopsPath, "stack", s.info.GopsAddr) s.execCmd(tarWriter, "gops.stats", s.info.GopsPath, "stats", s.info.GopsAddr) s.execCmd(tarWriter, "gops.memstats", s.info.GopsPath, "memstats", s.info.GopsAddr) + err = s.getPProf(tarWriter, "gops.pprof-heap") + if err != nil { + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithField("gops-path", s.info.GopsPath).WithError(err).Warn("Failed to dump gops pprof-heap") + } else { + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithField("gops-path", s.info.GopsPath).Info("Successfully dumped gops pprof-heap") + } } func (s *bugtoolInfo) dumpPolicyFilterMap(tarWriter *tar.Writer) error { From 045090ed8482ae784cfa700bf614de8a7bc426ae Mon Sep 17 00:00:00 2001 From: Djalal Harouni Date: Mon, 22 Jan 2024 20:25:36 +0100 Subject: [PATCH 3/4] tarball: copy gops into tarball [Upstream main 558d8693e2a3c39] Signed-off-by: Djalal Harouni --- Dockerfile.tarball | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile.tarball b/Dockerfile.tarball index 6339ff07356..114031114d7 100644 --- a/Dockerfile.tarball +++ b/Dockerfile.tarball @@ -21,6 +21,7 @@ COPY --from=tetragon-release /usr/bin/tetra /usr/local/bin/ # Helper binaries COPY --from=tetragon-release /usr/bin/bpftool /usr/local/lib/tetragon/ +COPY --from=tetragon-release /usr/bin/gops /usr/local/lib/tetragon/ # BPF files COPY --from=tetragon-release /var/lib/tetragon /usr/local/lib/tetragon/bpf From e15d5a1f5a359ebccdde03727956558304076513 Mon Sep 17 00:00:00 2001 From: Djalal Harouni Date: Thu, 1 Feb 2024 22:39:13 +0100 Subject: [PATCH 4/4] ci:tests: add tarball package CI tests This introduces tarball tests for 0.10, also it tries to test bugtool gops pprof-heap. Signed-off-by: Djalal Harouni --- .github/workflows/packages-e2e-tests.yaml | 90 +++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 .github/workflows/packages-e2e-tests.yaml diff --git a/.github/workflows/packages-e2e-tests.yaml b/.github/workflows/packages-e2e-tests.yaml new file mode 100644 index 00000000000..dd8e04fc410 --- /dev/null +++ b/.github/workflows/packages-e2e-tests.yaml @@ -0,0 +1,90 @@ +name: Packages e2e Tests + +on: + pull_request: + paths-ignore: + - "**.md" + - 'docs/**' + +jobs: + standalone-tests: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + # We use the native arch build + - os: ubuntu-22.04 + arch: amd64 + match_arch: x86-64 + + steps: + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + - name: Checkout Source Code + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + persist-credentials: false + fetch-depth: 0 + submodules: true + + - name: Getting version tag + id: tag + run: echo "tag=$(make version)" >> $GITHUB_OUTPUT + + # Logging into Docker Hub makes the job less flaky, but it relies on repo secrets so + # does not work for external contributors. Let's compromise and do this for internal + # PRs at least (for now). We can always improve the situation later. + - name: Login to Docker Hub + if: ${{ github.event.pull_request.head.repo.full_name == 'cilium/tetragon' }} + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME_CI }} + password: ${{ secrets.DOCKER_HUB_PASSWORD_CI }} + + - name: Generate Tetragon Tarball + id: tetragon-tarball + run: make tarball + + - name: Configure gops + run: | + sudo mkdir -p /etc/tetragon/tetragon.conf.d/ + echo "localhost:8118" | sudo tee /etc/tetragon/tetragon.conf.d/gops-address + + - name: Install Tetragon Tarball + run: | + tar zxvf tetragon-${{ steps.tag.outputs.tag }}-${{ matrix.arch }}.tar.gz + sudo ./tetragon-${{ steps.tag.outputs.tag }}-${{ matrix.arch }}/install.sh + working-directory: ./build/${{ matrix.arch }}/linux-tarball/ + + - name: Wait for Tetragon service + uses: nick-fields/retry@14672906e672a08bd6eeb15720e9ed3ce869cdd4 # v2 + with: + timeout_seconds: 30 + max_attempts: 5 + retry_wait_seconds: 5 + retry_on: error + command: | + # Ensure that default native builds work + file /usr/local/bin/tetragon | grep ${{ matrix.match_arch }} - + sudo systemctl is-active tetragon + sudo tetra status + + - name: Check Tetragon startup logs + run: sudo journalctl -b -u tetragon --no-pager + + - name: Test Tetragon + run: | + sudo tetra status + sudo grep "tetra" /var/log/tetragon/tetragon.log + sudo tetra tracingpolicy list + sudo tetra bugtool + test $(stat -c %a /var/run/tetragon/tetragon.sock) -eq "660" + sudo tetra bugtool 2>&1 | grep "Successfully dumped gops pprof-heap" - + + - name: Uninstall Tetragon Tarball + run: | + sudo ./tetragon-${{ steps.tag.outputs.tag }}-${{ matrix.arch }}/uninstall.sh + working-directory: ./build/${{ matrix.arch }}/linux-tarball/