enforcer: track missed notifications

api/v1/tetragon/tetragon.proto

@@ -473,6 +473,8 @@ enum KprobeAction {
 	KPROBE_ACTION_UNTRACKSOCK = 12;
     // NotifyEnforcer action notifies enforcer sensor.
 	KPROBE_ACTION_NOTIFYENFORCER = 13;
+    // CleanupEnforcerNotification action cleanups any state left by NotifyEnforcer
+	KPROBE_ACTION_CLEANUPENFORCERNOTIFICATION = 14;
 }
 
 message ProcessKprobe {

bpf/Makefile

@@ -109,14 +109,14 @@ $(DEPSDIR)bpf_enforcer.d: process/bpf_enforcer.c
 objs/bpf_multi_enforcer.ll: process/bpf_enforcer.c
 	$(CLANG) $(CLANG_FLAGS) -D__BPF_OVERRIDE_RETURN -D__MULTI_KPROBE -c $< -o $@
 
-$(DEPSDIR)/bpf_multi_enforcer.d: process/bpf_enforcer.c
+$(DEPSDIR)bpf_multi_enforcer.d: process/bpf_enforcer.c
 	$(CLANG) $(CLANG_FLAGS) -D__BPF_OVERRIDE_RETURN -D__MULTI_KPROBE -MM -MP -MT $(patsubst $(DEPSDIR)%.d, $(OBJSDIR)%.ll, $@) $< > $@
 
 ## bpf_fmodret_enforcer no bpf_override_return: we need fmod_ret
 objs/bpf_fmodret_enforcer.ll: process/bpf_enforcer.c
 	$(CLANG) $(CLANG_FLAGS) -c $< -o $@
 
-$(DEPSDIR)/bpf_fmodret_enforcer.d: process/bpf_enforcer.c
+$(DEPSDIR)bpf_fmodret_enforcer.d: process/bpf_enforcer.c
 	$(CLANG) $(CLANG_FLAGS) -MM -MP -MT $(patsubst $(DEPSDIR)%.d, $(OBJSDIR)%.ll, $@) $< > $@
 
 # PROCESSDIR

bpf/lib/generic.h

@@ -33,6 +33,9 @@ struct msg_selector_data {
 #endif
 };
 
+/* value to mask an offsset into msg_generic_kprobe->args */
+#define GENERIC_MSG_ARGS_MASK 0x7ff
+
 struct msg_generic_kprobe {
 	struct msg_common common;
 	struct msg_execve_key current;

bpf/process/bpf_enforcer.h

@@ -7,10 +7,17 @@
 #include "vmlinux.h"
 #include "bpf_helpers.h"
 
+/* information to track how an enforcer notify action was triggered */
+struct enforcer_act_info {
+	__u32 func_id;
+	__u32 arg;
+} __attribute__((packed));
+
 struct enforcer_data {
 	__s16 error;
 	__s16 signal;
-};
+	struct enforcer_act_info act_info;
+} __attribute__((packed));
 
 struct {
 	__uint(type, BPF_MAP_TYPE_HASH);
@@ -19,18 +26,81 @@ struct {
 	__type(value, struct enforcer_data);
 } enforcer_data SEC(".maps");
 
-FUNC_INLINE void do_enforcer_action(int error, int signal)
+enum enforcer_missed_reason {
+	ENFORCER_MISSED_OVERWRITTEN = 1,
+	ENFORCER_MISSED_NOACTION = 2,
+};
+
+struct enforcer_missed_key {
+	struct enforcer_act_info act_info;
+	__u32 reason; // see enforcer_missed_reason for values
+} __attribute__((packed));
+
+/* map to keep track of missed notifications */
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__uint(max_entries, 128);
+	__type(key, struct enforcer_missed_key);
+	__type(value, __u32);
+} enforcer_missed_notifications SEC(".maps");
+
+FUNC_INLINE void
+enforcer_update_missed_notifications(struct enforcer_missed_key *key)
+{
+	int err;
+	__u32 *counter = map_lookup_elem(&enforcer_missed_notifications, key), one = 1;
+
+	if (counter) {
+		__sync_fetch_and_add(counter, one);
+		return;
+	}
+
+	err = map_update_elem(&enforcer_missed_notifications, key, &one, BPF_NOEXIST);
+	if (!err)
+		return;
+
+	/* in case we raced with another thread and an entry was already created, retry to do a
+	 * lookup
+	 */
+	counter = map_lookup_elem(&enforcer_missed_notifications, key);
+	if (counter) {
+		__sync_fetch_and_add(counter, one);
+	}
+}
+
+FUNC_INLINE void do_enforcer_cleanup(void)
+{
+	struct enforcer_data *ptr;
+	__u64 id = get_current_pid_tgid();
+
+	ptr = map_lookup_elem(&enforcer_data, &id);
+	if (ptr) {
+		struct enforcer_missed_key missed_key = {
+			.act_info = ptr->act_info,
+			.reason = ENFORCER_MISSED_NOACTION,
+		};
+		enforcer_update_missed_notifications(&missed_key);
+	}
+}
+
+FUNC_INLINE void do_enforcer_action(int error, int signal, struct enforcer_act_info act_info)
 {
 	__u64 id = get_current_pid_tgid();
 	struct enforcer_data *ptr, data = {
 		.error = (__s16)error,
 		.signal = (__s16)signal,
+		.act_info = act_info,
 	};
 
 	ptr = map_lookup_elem(&enforcer_data, &id);
 	if (ptr) {
-		ptr->error = (__s16)error;
-		ptr->signal = (__s16)signal;
+		/* there is another entry already, update enforcer_missed_notifications */
+		struct enforcer_missed_key missed_key = {
+			.act_info = ptr->act_info,
+			.reason = ENFORCER_MISSED_OVERWRITTEN,
+		};
+		enforcer_update_missed_notifications(&missed_key);
+		*ptr = data;
 	} else {
 		map_update_elem(&enforcer_data, &id, &data, BPF_ANY);
 	}

bpf/process/types/basic.h

@@ -109,6 +109,7 @@ enum {
 	ACTION_TRACKSOCK = 10,
 	ACTION_UNTRACKSOCK = 11,
 	ACTION_NOTIFY_ENFORCER = 12,
+	ACTION_CLEANUP_ENFORCER_NOTIFICATION = 13,
 };
 
 enum {
@@ -1868,6 +1869,20 @@ installfd(struct msg_generic_kprobe *e, int fd, int name, bool follow)
 	return err;
 }
 
+FUNC_INLINE __u64
+msg_generic_arg_value_u64(struct msg_generic_kprobe *e, unsigned int arg_id, __u64 err_val)
+{
+	__u32 argoff;
+	__u64 *ret;
+
+	if (arg_id > MAX_POSSIBLE_ARGS)
+		return err_val;
+	argoff = e->argsoff[arg_id];
+	argoff &= GENERIC_MSG_ARGS_MASK;
+	ret = (__u64 *)&e->args[argoff];
+	return *ret;
+}
+
 FUNC_INLINE int
 copyfd(struct msg_generic_kprobe *e, int oldfd, int newfd)
 {
@@ -2143,12 +2158,18 @@ struct {
 } stack_trace_map SEC(".maps");
 
 #if defined GENERIC_TRACEPOINT || defined GENERIC_KPROBE
-FUNC_INLINE void do_action_notify_enforcer(int error, int signal)
+FUNC_INLINE void do_action_notify_enforcer(struct msg_generic_kprobe *e,
+					   int error, int signal, int info_arg_id)
 {
-	do_enforcer_action(error, signal);
+	__u64 argv = msg_generic_arg_value_u64(e, info_arg_id, 0);
+	struct enforcer_act_info info = {
+		.func_id = e->func_id,
+		.arg = argv,
+	};
+	do_enforcer_action(error, signal, info);
 }
 #else
-#define do_action_notify_enforcer(error, signal)
+#define do_action_notify_enforcer(e, error, signal, info_arg_id)
 #endif
 
 FUNC_LOCAL __u32
@@ -2163,6 +2184,7 @@ do_action(void *ctx, __u32 i, struct selector_action *actions,
 	int fdi, namei;
 	int newfdi, oldfdi;
 	int socki;
+	int argi __maybe_unused;
 	int err = 0;
 	int zero = 0;
 	__u64 id;
@@ -2257,8 +2279,11 @@ do_action(void *ctx, __u32 i, struct selector_action *actions,
 	case ACTION_NOTIFY_ENFORCER:
 		error = actions->act[++i];
 		signal = actions->act[++i];
-		do_action_notify_enforcer(error, signal);
+		argi = actions->act[++i];
+		do_action_notify_enforcer(e, error, signal, argi);
 		break;
+	case ACTION_CLEANUP_ENFORCER_NOTIFICATION:
+		do_enforcer_cleanup();
 	default:
 		break;
 	}