Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

tetragon: Add map_errors_update_total/map_errors_delete_total metrics #3346

Merged
merged 2 commits into from
Jan 30, 2025
+63 −79
Merged

tetragon: Add map_errors_update_total/map_errors_delete_total metrics #3346

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
12cfc3b
tetragon: Store stats map update/delete errors
olsajiri Jan 17, 2025
9885fe8
tetragon: Add map_errors_update_total/map_errors_delete_total metrics
olsajiri Jan 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 28 additions & 48 deletions bpf/lib/process.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -371,18 +371,20 @@ struct {
__type(value, struct execve_map_value);
} execve_map SEC(".maps");

enum {
MAP_STATS_COUNT = 0,
MAP_STATS_EUPDATE = 1,
MAP_STATS_EDELETE = 2,
MAP_STATS_MAX = 3,
};

struct {
__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
__uint(max_entries, 2);
__uint(max_entries, MAP_STATS_MAX);
__type(key, __s32);
__type(value, __s64);
} execve_map_stats SEC(".maps");

enum {
MAP_STATS_COUNT = 0,
MAP_STATS_ERROR = 1,
};

struct {
__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
__uint(max_entries, 1);
Expand Down Expand Up @@ -434,7 +436,7 @@ struct {
*/
struct {
__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
__uint(max_entries, 2);
__uint(max_entries, MAP_STATS_MAX);
__type(key, __s32);
__type(value, __s64);
} tg_execve_joined_info_map_stats SEC(".maps");
Expand All @@ -460,17 +462,18 @@ FUNC_INLINE int64_t validate_msg_execve_size(int64_t size)
return size;
}

// execve_map_error() will increment the map error counter
FUNC_INLINE void execve_map_error(void)
FUNC_INLINE void stats_update(struct bpf_map_def *map, __u32 key, int inc)
{
int one = MAP_STATS_ERROR;
__s64 *cntr;

cntr = map_lookup_elem(&execve_map_stats, &one);
cntr = map_lookup_elem(map, &key);
if (cntr)
*cntr = *cntr + 1;
*cntr = *cntr + inc;
}

#define STATS_INC(map, key) stats_update((struct bpf_map_def *)&(map), MAP_STATS_##key, 1)
#define STATS_DEC(map, key) stats_update((struct bpf_map_def *)&(map), MAP_STATS_##key, -1)

// execve_map_get will look up if pid exists and return it if it does. If it
// does not, it will create a new one and return it.
FUNC_INLINE struct execve_map_value *execve_map_get(__u32 pid)
Expand All @@ -481,7 +484,6 @@ FUNC_INLINE struct execve_map_value *execve_map_get(__u32 pid)
if (!event) {
struct execve_map_value *value;
int err, zero = MAP_STATS_COUNT;
__s64 *cntr;

value = map_lookup_elem(&execve_val, &zero);
if (!value)
Expand All @@ -490,11 +492,9 @@ FUNC_INLINE struct execve_map_value *execve_map_get(__u32 pid)
memset(value, 0, sizeof(struct execve_map_value));
err = map_update_elem(&execve_map, &pid, value, 0);
if (!err) {
cntr = map_lookup_elem(&execve_map_stats, &zero);
if (cntr)
*cntr = *cntr + 1;
STATS_INC(execve_map_stats, COUNT);
} else {
execve_map_error();
STATS_INC(execve_map_stats, EUPDATE);
}
event = map_lookup_elem(&execve_map, &pid);
}
Expand All @@ -509,61 +509,41 @@ FUNC_INLINE struct execve_map_value *execve_map_get_noinit(__u32 pid)
FUNC_INLINE void execve_map_delete(__u32 pid)
{
int err = map_delete_elem(&execve_map, &pid);
int zero = MAP_STATS_COUNT;
__s64 *cntr;

if (!err) {
cntr = map_lookup_elem(&execve_map_stats, &zero);
if (cntr)
*cntr = *cntr - 1;
STATS_DEC(execve_map_stats, COUNT);
} else {
execve_map_error();
STATS_INC(execve_map_stats, EDELETE);
}
}

// execve_joined_info_map_error() will increment the map error counter
FUNC_INLINE void execve_joined_info_map_error(void)
{
int one = MAP_STATS_ERROR;
__s64 *cntr;

cntr = map_lookup_elem(&tg_execve_joined_info_map_stats, &one);
if (cntr)
*cntr = *cntr + 1;
}

FUNC_INLINE void execve_joined_info_map_set(__u64 tid, struct execve_info *info)
{
int err, zero = MAP_STATS_COUNT;
__s64 *cntr;
int err;

err = map_update_elem(&tg_execve_joined_info_map, &tid, info, BPF_ANY);
if (err < 0) {
if (!err) {
STATS_INC(tg_execve_joined_info_map_stats, COUNT);
} else {
/* -EBUSY or -ENOMEM with the help of the cntr error
* on the stats map this can be a good indication of
* long running workloads and if we have to make the
* map size bigger for such cases.
*/
execve_joined_info_map_error();
return;
STATS_INC(tg_execve_joined_info_map_stats, EUPDATE);
}

cntr = map_lookup_elem(&tg_execve_joined_info_map_stats, &zero);
if (cntr)
*cntr = *cntr + 1;
}

/* Clear up some space for next threads */
FUNC_INLINE void execve_joined_info_map_clear(__u64 tid)
{
int err, zero = MAP_STATS_COUNT;
__s64 *cntr;
int err;

err = map_delete_elem(&tg_execve_joined_info_map, &tid);
if (!err) {
cntr = map_lookup_elem(&tg_execve_joined_info_map_stats, &zero);
if (cntr)
*cntr = *cntr - 1;
STATS_DEC(tg_execve_joined_info_map_stats, COUNT);
} else {
STATS_INC(tg_execve_joined_info_map_stats, EDELETE);
}
/* We don't care here about -ENOENT as there is no guarantee entries
* will be present anyway.
Expand Down
2 changes: 1 addition & 1 deletion contrib/upgrade-notes/latest.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,4 @@ Depending on your setup, changes listed here might require a manual intervention

### Metrics

* TBD
* `tetragon_map_errors_total` metric is replaced by `map_errors_update_total` and `map_errors_delete_total`.
12 changes: 10 additions & 2 deletions docs/content/en/docs/reference/metrics.md
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 8 additions & 3 deletions pkg/metrics/mapmetrics/mapmetrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,14 @@ var (
"Capacity of a BPF map. Expected to be constant.",
nil, []metrics.ConstrainedLabel{MapLabel}, nil,
))
MapErrors = metrics.MustNewCustomGauge(metrics.NewOpts(
consts.MetricsNamespace, "", "map_errors_total",
"The number of errors per map.",
MapErrorsUpdate = metrics.MustNewCustomGauge(metrics.NewOpts(
consts.MetricsNamespace, "", "map_errors_update_total",
"The number of failed updates per map.",
nil, []metrics.ConstrainedLabel{MapLabel}, nil,
))
MapErrorsDelete = metrics.MustNewCustomGauge(metrics.NewOpts(
consts.MetricsNamespace, "", "map_errors_delete_total",
"The number of failed deletes per map.",
nil, []metrics.ConstrainedLabel{MapLabel}, nil,
))
)
41 changes: 16 additions & 25 deletions pkg/observer/observer_stats.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ func NewBPFCollector() metrics.CollectorWithInit {
metrics.CustomMetrics{
mapmetrics.MapSize,
mapmetrics.MapCapacity,
mapmetrics.MapErrors,
mapmetrics.MapErrorsUpdate,
mapmetrics.MapErrorsDelete,
},
collect,
collectForDocs,
Expand Down Expand Up @@ -72,51 +73,41 @@ func collect(ch chan<- prometheus.Metric) {
}
defer mapLink.Close()

updateMapSize(ch, mapLinkStats, name)
ch <- mapmetrics.MapCapacity.MustMetric(
float64(mapLink.MaxEntries()),
name,
)
updateMapErrors(ch, mapLinkStats, name)
update(mapLinkStats, 0, func(sum float64) {
ch <- mapmetrics.MapSize.MustMetric(sum, name)
})
update(mapLinkStats, 1, func(sum float64) {
ch <- mapmetrics.MapErrorsUpdate.MustMetric(sum, name)
})
update(mapLinkStats, 2, func(sum float64) {
ch <- mapmetrics.MapErrorsDelete.MustMetric(sum, name)
})
}
}

func updateMapSize(ch chan<- prometheus.Metric, mapLinkStats *ebpf.Map, name string) {
func update(mapLinkStats *ebpf.Map, key int32, update func(sum float64)) {
var values []int64
if err := mapLinkStats.Lookup(int32(0), &values); err != nil {
return
}

sum := int64(0)
for _, n := range values {
sum += n
}
ch <- mapmetrics.MapSize.MustMetric(
float64(sum),
name,
)
}

func updateMapErrors(ch chan<- prometheus.Metric, mapLinkStats *ebpf.Map, name string) {
var values []int64
if err := mapLinkStats.Lookup(int32(1), &values); err != nil {
if err := mapLinkStats.Lookup(key, &values); err != nil {
return
}

sum := int64(0)
for _, n := range values {
sum += n
}
ch <- mapmetrics.MapErrors.MustMetric(
float64(sum),
name,
)
update(float64(sum))
}

func collectForDocs(ch chan<- prometheus.Metric) {
for _, m := range mapmetrics.MapLabel.Values {
ch <- mapmetrics.MapSize.MustMetric(0, m)
ch <- mapmetrics.MapCapacity.MustMetric(0, m)
ch <- mapmetrics.MapErrors.MustMetric(0, m)
ch <- mapmetrics.MapErrorsUpdate.MustMetric(0, m)
ch <- mapmetrics.MapErrorsDelete.MustMetric(0, m)
}
}
Loading