diff --git a/.github/workflows/packages-e2e-tests.yaml b/.github/workflows/packages-e2e-tests.yaml new file mode 100644 index 00000000000..dd8e04fc410 --- /dev/null +++ b/.github/workflows/packages-e2e-tests.yaml @@ -0,0 +1,90 @@ +name: Packages e2e Tests + +on: + pull_request: + paths-ignore: + - "**.md" + - 'docs/**' + +jobs: + standalone-tests: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + # We use the native arch build + - os: ubuntu-22.04 + arch: amd64 + match_arch: x86-64 + + steps: + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + - name: Checkout Source Code + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + persist-credentials: false + fetch-depth: 0 + submodules: true + + - name: Getting version tag + id: tag + run: echo "tag=$(make version)" >> $GITHUB_OUTPUT + + # Logging into Docker Hub makes the job less flaky, but it relies on repo secrets so + # does not work for external contributors. Let's compromise and do this for internal + # PRs at least (for now). We can always improve the situation later. + - name: Login to Docker Hub + if: ${{ github.event.pull_request.head.repo.full_name == 'cilium/tetragon' }} + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME_CI }} + password: ${{ secrets.DOCKER_HUB_PASSWORD_CI }} + + - name: Generate Tetragon Tarball + id: tetragon-tarball + run: make tarball + + - name: Configure gops + run: | + sudo mkdir -p /etc/tetragon/tetragon.conf.d/ + echo "localhost:8118" | sudo tee /etc/tetragon/tetragon.conf.d/gops-address + + - name: Install Tetragon Tarball + run: | + tar zxvf tetragon-${{ steps.tag.outputs.tag }}-${{ matrix.arch }}.tar.gz + sudo ./tetragon-${{ steps.tag.outputs.tag }}-${{ matrix.arch }}/install.sh + working-directory: ./build/${{ matrix.arch }}/linux-tarball/ + + - name: Wait for Tetragon service + uses: nick-fields/retry@14672906e672a08bd6eeb15720e9ed3ce869cdd4 # v2 + with: + timeout_seconds: 30 + max_attempts: 5 + retry_wait_seconds: 5 + retry_on: error + command: | + # Ensure that default native builds work + file /usr/local/bin/tetragon | grep ${{ matrix.match_arch }} - + sudo systemctl is-active tetragon + sudo tetra status + + - name: Check Tetragon startup logs + run: sudo journalctl -b -u tetragon --no-pager + + - name: Test Tetragon + run: | + sudo tetra status + sudo grep "tetra" /var/log/tetragon/tetragon.log + sudo tetra tracingpolicy list + sudo tetra bugtool + test $(stat -c %a /var/run/tetragon/tetragon.sock) -eq "660" + sudo tetra bugtool 2>&1 | grep "Successfully dumped gops pprof-heap" - + + - name: Uninstall Tetragon Tarball + run: | + sudo ./tetragon-${{ steps.tag.outputs.tag }}-${{ matrix.arch }}/uninstall.sh + working-directory: ./build/${{ matrix.arch }}/linux-tarball/ diff --git a/Dockerfile.tarball b/Dockerfile.tarball index 6339ff07356..114031114d7 100644 --- a/Dockerfile.tarball +++ b/Dockerfile.tarball @@ -21,6 +21,7 @@ COPY --from=tetragon-release /usr/bin/tetra /usr/local/bin/ # Helper binaries COPY --from=tetragon-release /usr/bin/bpftool /usr/local/lib/tetragon/ +COPY --from=tetragon-release /usr/bin/gops /usr/local/lib/tetragon/ # BPF files COPY --from=tetragon-release /var/lib/tetragon /usr/local/lib/tetragon/bpf diff --git a/cmd/tetra/bugtool/bugtool.go b/cmd/tetra/bugtool/bugtool.go index 93e7ccd73a8..e7d4eb41b00 100644 --- a/cmd/tetra/bugtool/bugtool.go +++ b/cmd/tetra/bugtool/bugtool.go @@ -23,6 +23,7 @@ import ( var ( outFile string bpfTool string + gops string ) func New() *cobra.Command { @@ -30,12 +31,13 @@ func New() *cobra.Command { Use: "bugtool", Short: "Produce a tar archive with debug information", Run: func(cmd *cobra.Command, args []string) { - bugtool.Bugtool(outFile, bpfTool) + bugtool.Bugtool(outFile, bpfTool, gops) }, } flags := bugtoolCmd.Flags() flags.StringVarP(&outFile, "out", "o", "tetragon-bugtool.tar.gz", "Output filename") flags.StringVar(&bpfTool, "bpftool", "", "Path to bpftool binary") + flags.StringVar(&gops, "gops", "", "Path to gops binary") return bugtoolCmd } diff --git a/pkg/bugtool/bugtool.go b/pkg/bugtool/bugtool.go index 389a7dcd5c7..759e0a142d3 100644 --- a/pkg/bugtool/bugtool.go +++ b/pkg/bugtool/bugtool.go @@ -14,6 +14,7 @@ import ( "errors" "fmt" "io" + "net" "net/http" "os" "os/exec" @@ -26,6 +27,7 @@ import ( "github.com/cilium/tetragon/pkg/defaults" "github.com/cilium/tetragon/pkg/logger" "github.com/cilium/tetragon/pkg/policyfilter" + gopssignal "github.com/google/gops/signal" "go.uber.org/multierr" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -44,6 +46,7 @@ type InitInfo struct { GopsAddr string `json:"gops_address"` MapDir string `json:"map_dir"` BpfToolPath string `json:"bpftool_path"` + GopsPath string `json:"gops_path"` } // LoadInitInfo returns the InitInfo by reading the info file from its default location @@ -83,6 +86,14 @@ func doSaveInitInfo(fname string, info *InitInfo) error { logger.GetLogger().WithField("bpftool", info.BpfToolPath).Info("Successfully detected bpftool path") } + gops, err := exec.LookPath("gops") + if err != nil { + logger.GetLogger().Warn("failed to locate gops binary, on bugtool debugging ensure you have gops installed") + } else { + info.GopsPath = gops + logger.GetLogger().WithField("gops", info.GopsPath).Info("Successfully detected gops path") + } + // Create DefaultRunDir if it does not already exist if err := os.MkdirAll(defaults.DefaultRunDir, 0755); err != nil { logger.GetLogger().WithField("infoFile", fname).Warn("failed to directory exists") @@ -182,7 +193,7 @@ func (s *bugtoolInfo) tarAddFile(tarWriter *tar.Writer, fnameSrc string, fnameDs } // Bugtool gathers information and writes it as a tar archive in the given filename -func Bugtool(outFname string, bpftool string) error { +func Bugtool(outFname string, bpftool string, gops string) error { info, err := LoadInitInfo() if err != nil { return err @@ -192,6 +203,10 @@ func Bugtool(outFname string, bpftool string) error { info.BpfToolPath = bpftool } + if gops != "" { + info.GopsPath = gops + } + return doBugtool(info, outFname) } @@ -488,7 +503,7 @@ func (s *bugtoolInfo) addBpftoolInfo(tarWriter *tar.Writer) { _, err := os.Stat(s.info.BpfToolPath) if err != nil { - s.multiLog.WithError(err).Warn("Failed to locate bpftool, please install it.") + s.multiLog.WithError(err).Warn("Failed to locate bpftool. Please install it or specify its path, see 'bugtool --help'") return } s.execCmd(tarWriter, "bpftool-maps.json", s.info.BpfToolPath, "map", "show", "-j") @@ -496,13 +511,61 @@ func (s *bugtoolInfo) addBpftoolInfo(tarWriter *tar.Writer) { s.execCmd(tarWriter, "bpftool-cgroups.json", s.info.BpfToolPath, "cgroup", "tree", "-j") } +func (s *bugtoolInfo) getPProf(tarWriter *tar.Writer, file string) error { + if s.info.GopsAddr == "" { + s.multiLog.Info("Skipping gops dump info as daemon is running without gops, use --gops-address to enable gops") + return nil + } + + s.multiLog.WithField("gops-address", s.info.GopsAddr).Info("Contacting gops server for pprof dump") + + conn, err := net.Dial("tcp", s.info.GopsAddr) + if err != nil { + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to contact gops server") + return err + } + + buf := []byte{gopssignal.HeapProfile} + if _, err := conn.Write(buf); err != nil { + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to send gops pprof-heap command") + return err + } + + buff := new(bytes.Buffer) + if _, err = buff.ReadFrom(conn); err != nil { + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed reading gops pprof-heap response") + } + return s.tarAddBuff(tarWriter, file, buff) +} + func (s *bugtoolInfo) addGopsInfo(tarWriter *tar.Writer) { if s.info.GopsAddr == "" { + s.multiLog.Info("Skipping gops dump info as daemon is running without gops, use --gops-address to enable gops") return } - s.execCmd(tarWriter, "gops.stack", "gops", "stack", s.info.GopsAddr) - s.execCmd(tarWriter, "gpos.stats", "gops", "stats", s.info.GopsAddr) - s.execCmd(tarWriter, "gops.memstats", "gops", "memstats", s.info.GopsAddr) + + if s.info.GopsPath == "" { + s.multiLog.WithField("gops-address", s.info.GopsAddr).Warn("Failed to locate gops. Please install it or specify its path, see 'bugtool --help'") + return + } + + _, err := os.Stat(s.info.GopsPath) + if err != nil { + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithError(err).Warn("Failed to locate gops, please install it") + return + } + + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithField("gops-path", s.info.GopsPath).Info("Dumping gops information") + + s.execCmd(tarWriter, "gops.stack", s.info.GopsPath, "stack", s.info.GopsAddr) + s.execCmd(tarWriter, "gops.stats", s.info.GopsPath, "stats", s.info.GopsAddr) + s.execCmd(tarWriter, "gops.memstats", s.info.GopsPath, "memstats", s.info.GopsAddr) + err = s.getPProf(tarWriter, "gops.pprof-heap") + if err != nil { + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithField("gops-path", s.info.GopsPath).WithError(err).Warn("Failed to dump gops pprof-heap") + } else { + s.multiLog.WithField("gops-address", s.info.GopsAddr).WithField("gops-path", s.info.GopsPath).Info("Successfully dumped gops pprof-heap") + } } func (s *bugtoolInfo) dumpPolicyFilterMap(tarWriter *tar.Writer) error { diff --git a/pkg/observer/observer_test_helper.go b/pkg/observer/observer_test_helper.go index 2aab6bf84ea..ef9bca684d1 100644 --- a/pkg/observer/observer_test_helper.go +++ b/pkg/observer/observer_test_helper.go @@ -129,7 +129,7 @@ func withNotestfail(notestfail bool) TestOption { func testDone(t *testing.T, obs *Observer) { if t.Failed() { bugtoolFname := "/tmp/tetragon-bugtool.tar.gz" - if err := bugtool.Bugtool(bugtoolFname, ""); err == nil { + if err := bugtool.Bugtool(bugtoolFname, "", ""); err == nil { logger.GetLogger().WithField("test", t.Name()). WithField("file", bugtoolFname).Info("Dumped bugtool info") } else {