From 044cc3793093f236af05d7155d291206f8456bb1 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Mon, 2 Nov 2026 18:16:30 +0200 Subject: [PATCH 01/34] feat(enforce_privilege): BPF LSM privilege gadget and common enforcement library pkg/ebpf/gadgets/common is a small library for BPF LSM enforcement gadgets: the bindings map (cgroup -> policy slot), policy rank resolution, verdict base fields and counters, the arming rule, and the Go binding controller of one node-wide instance. It is taken from the enforce_exec simple engine of the pinned Inspektor Gadget fork. pkg/ebpf/gadgets/enforce_privilege enforces the SandboxPolicy privilege section with BPF LSM, no seccomp: namespaces, mounts, pivot_root/chroot, capabilities, ptrace, modules, bpf, perf, keys, io_uring, userfaultfd, mknod, setuid/setgid and capset. Its compiler merges several policies into one slot per combination. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- Makefile | 2 +- pkg/ebpf/gadgets/common/README.md | 32 + pkg/ebpf/gadgets/common/arming.h | 117 +++ pkg/ebpf/gadgets/common/bindings.h | 63 ++ .../gadgets/common/enforcement/arm_linux.go | 120 +++ .../gadgets/common/enforcement/bindings.go | 311 +++++++ .../common/enforcement/bindings_test.go | 154 ++++ pkg/ebpf/gadgets/common/policy.h | 71 ++ pkg/ebpf/gadgets/common/verdict.h | 95 ++ pkg/ebpf/gadgets/enforce_privilege/Makefile | 8 + pkg/ebpf/gadgets/enforce_privilege/README.md | 108 +++ .../gadgets/enforce_privilege/gadget.yaml | 24 + .../gadgets/enforce_privilege/policy/api.go | 24 + .../enforce_privilege/policy/kernel.go | 332 +++++++ .../enforce_privilege/policy/kernel_test.go | 249 ++++++ .../enforce_privilege/policy/policy.go | 132 +++ .../enforce_privilege/privilege_policy.h | 84 ++ .../gadgets/enforce_privilege/program.bpf.c | 823 ++++++++++++++++++ .../enforce_privilege/runtime/runtime.go | 132 +++ .../enforce_privilege/runtime/runtime_test.go | 65 ++ 20 files changed, 2945 insertions(+), 1 deletion(-) create mode 100644 pkg/ebpf/gadgets/common/README.md create mode 100644 pkg/ebpf/gadgets/common/arming.h create mode 100644 pkg/ebpf/gadgets/common/bindings.h create mode 100644 pkg/ebpf/gadgets/common/enforcement/arm_linux.go create mode 100644 pkg/ebpf/gadgets/common/enforcement/bindings.go create mode 100644 pkg/ebpf/gadgets/common/enforcement/bindings_test.go create mode 100644 pkg/ebpf/gadgets/common/policy.h create mode 100644 pkg/ebpf/gadgets/common/verdict.h create mode 100644 pkg/ebpf/gadgets/enforce_privilege/Makefile create mode 100644 pkg/ebpf/gadgets/enforce_privilege/README.md create mode 100644 pkg/ebpf/gadgets/enforce_privilege/gadget.yaml create mode 100644 pkg/ebpf/gadgets/enforce_privilege/policy/api.go create mode 100644 pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go create mode 100644 pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go create mode 100644 pkg/ebpf/gadgets/enforce_privilege/policy/policy.go create mode 100644 pkg/ebpf/gadgets/enforce_privilege/privilege_policy.h create mode 100644 pkg/ebpf/gadgets/enforce_privilege/program.bpf.c create mode 100644 pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go create mode 100644 pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go diff --git a/Makefile b/Makefile index 231f2725a7..8fea1ceb69 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ IMAGE?=quay.io/kubescape/$(BINARY_NAME) # resolver can resolve relative opens against their dirfd/cwd. GADGETS=advise_seccomp trace_capabilities trace_dns trace_exec VERSION=v0.48.1 -KUBESCAPE_GADGETS=bpf exit fork hardlink http iouring_new iouring_old kmod network ptrace randomx ssh symlink unshare +KUBESCAPE_GADGETS=bpf enforce_privilege exit fork hardlink http iouring_new iouring_old kmod network ptrace randomx ssh symlink unshare # BUILT_GADGETS are vendored under pkg/ebpf/gadgets and built under their full # upstream image name+tag so node-agent's pinned openImageName keeps resolving. BUILT_GADGETS=trace_open diff --git a/pkg/ebpf/gadgets/common/README.md b/pkg/ebpf/gadgets/common/README.md new file mode 100644 index 0000000000..0359e1792f --- /dev/null +++ b/pkg/ebpf/gadgets/common/README.md @@ -0,0 +1,32 @@ +# Common enforcement library + +Shared pieces of the BPF LSM enforcement gadgets (`enforce_privilege` today). +They are taken from the enforce_exec gadget of the pinned Inspektor Gadget fork +and keep the first verdict fields, the mode encoding, the `counters` slots, the +identity type and the `bindings` key and value size. To move enforce_exec onto +the library, these still differ and need a decision: the `bindings` value is a +policy slot here (one instance per node) but 0 or a runtime mount namespace +inode in enforce_exec (one instance per container, restricted bindings); +enforce_exec reports allowed execs with +the policy mode (this library reports only denies), counts every allowed +decision, has restricted bindings and kernel-fault checks in its controller, +and keeps its path rules in an array that `policy.h` does not describe. + +| File | Content | +|---|---| +| `bindings.h` | `bindings` map: cgroup id → the container's policy slot; optional parent-cgroup depth | +| `policy.h` | policy modes (0 off, 1 audit, 2 enforce), rank resolution (explicit deny > allow > default, one mode per rule), shared policy value | +| `verdict.h` | first verdict fields (`generation`, `cgroup`, `deny`, `mode`), per-CPU `counters` (denied, submitted, dropped, allowed, section error, rate limited), 100 reports per CPU per second | +| `arming.h` | arming rule: judge only tasks that executed outside the runtime mount namespace, and their children | +| `enforcement/` | Go: the binding controller of one node-wide instance (policy written and frozen before any binding, live `Bind`/`Unbind` of a container to a slot, readiness, arming of tasks already running) | + +Section-specific parts (the policy key layout, hooks, compiler) stay in each +gadget. + +## Build + +`ig image build` mounts only the gadget folder, so `#include "../common/..."` +does not resolve by default. A gadget Makefile sets `IG_SOURCE_PATH` to the +repository root; the builder then mounts the whole repository. This reuses an +Inspektor Gadget variable meant for in-tree gadgets of the Inspektor Gadget +repository. diff --git a/pkg/ebpf/gadgets/common/arming.h b/pkg/ebpf/gadgets/common/arming.h new file mode 100644 index 0000000000..4dda642b59 --- /dev/null +++ b/pkg/ebpf/gadgets/common/arming.h @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright 2026 The Kubescape Authors + +// Arming rule shared by the SandboxPolicy prevention gadgets. +// +// A cgroup-scoped enforcer also sees the container runtime's own setup inside +// the container cgroup (runc init creates namespaces, mounts, pivots root and, +// for exec, joins namespaces). Seccomp never sees that work because runc +// applies the filter as its last step, right before it execs the workload. +// +// This header gives the same boundary to BPF: a task is "armed" when it +// executes a program outside the host mount namespace, and a child inherits +// the mark at fork. Only armed tasks are judged. Process names are never +// trusted. Tasks that already ran before the gadget loaded are armed from user +// space by writing the task storage map with a pidfd key. +// +// runc init for "exec" execs inside the container cgroup but still in the host +// mount namespace, so it is not armed. The command it execs after joining the +// container mount namespace is armed. + +#ifndef __ENFORCEMENT_ARMING_H +#define __ENFORCEMENT_ARMING_H + +#include +#include +#include + +// Inode number of the mount namespace the container runtime runs in. 0 means +// the mount namespace of the global init task. Set it when the runtime does not +// run in init's mount namespace (for example a kind node). +const volatile __u64 host_mntns_id = 0; + +struct { + __uint(type, BPF_MAP_TYPE_TASK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, __u32); +} armed_tasks SEC(".maps"); + +// A thread counts as armed when it or its thread group leader is armed. User +// space can arm only the leader on kernels without PIDFD_THREAD (< 6.9). +static __always_inline bool arming_task_armed(struct task_struct *task) +{ + __u32 *a = bpf_task_storage_get(&armed_tasks, task, 0, 0); + struct task_struct *leader; + + if (a && *a) + return true; + leader = task->group_leader; + if (!leader || leader == task) + return false; + a = bpf_task_storage_get(&armed_tasks, leader, 0, 0); + return a && *a; +} + +static __always_inline void arming_mark(struct task_struct *task) +{ + __u32 *a = bpf_task_storage_get(&armed_tasks, task, 0, + BPF_LOCAL_STORAGE_GET_F_CREATE); + if (a) + *a = 1; +} + +#define ARMING_MAX_ANCESTORS 64 + +// Cached mount namespace of the global init task (0 = not known yet). +__u64 arming_init_mntns = 0; + +// arming_host_mntns returns host_mntns_id, or the mount namespace of the global +// init task. The real_parent chain of every task ends at the global init task +// (container processes are reparented to the runtime shim, which descends from +// it). +static __always_inline __u64 arming_host_mntns(struct task_struct *task) +{ + struct task_struct *t = task; + + if (host_mntns_id) + return host_mntns_id; + if (arming_init_mntns) + return arming_init_mntns; + + for (int i = 0; i < ARMING_MAX_ANCESTORS; i++) { + struct task_struct *parent = BPF_CORE_READ(t, real_parent); + + if (!parent || parent == t) { + // Reached the top: cache the init task's namespace. + arming_init_mntns = + BPF_CORE_READ(t, nsproxy, mnt_ns, ns.inum); + return arming_init_mntns; + } + t = parent; + } + // Tree deeper than the walk: do not cache a value that may be wrong. + return BPF_CORE_READ(t, nsproxy, mnt_ns, ns.inum); +} + +// arming_on_exec runs after a successful exec (bprm_committed_creds). +static __always_inline void arming_on_exec(void) +{ + struct task_struct *task = bpf_get_current_task_btf(); + __u64 mntns = BPF_CORE_READ(task, nsproxy, mnt_ns, ns.inum); + + if (arming_task_armed(task)) + return; + if (mntns == arming_host_mntns(task)) + return; + arming_mark(task); +} + +// arming_on_fork runs in task_alloc: the child inherits the parent's mark. +static __always_inline void arming_on_fork(struct task_struct *child) +{ + if (arming_task_armed(bpf_get_current_task_btf())) + arming_mark(child); +} + +#endif /* __ENFORCEMENT_ARMING_H */ diff --git a/pkg/ebpf/gadgets/common/bindings.h b/pkg/ebpf/gadgets/common/bindings.h new file mode 100644 index 0000000000..c3101978e3 --- /dev/null +++ b/pkg/ebpf/gadgets/common/bindings.h @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright 2026 The Kubescape Authors + +// Container binding map of a node-wide enforcement gadget instance: +// key: cgroup v2 id +// value: the policy slot of the container (one combination of policies) +// Same name, key and value size as the enforce_exec `bindings` map of the +// Inspektor Gadget fork. The value meaning differs: there it is 0 or a runtime +// mount namespace inode (restricted binding). + +#ifndef __ENFORCEMENT_BINDINGS_H +#define __ENFORCEMENT_BINDINGS_H + +#include +#include +#include + +#define BINDINGS_MAX 1024 + +// Number of cgroup levels checked: the task's cgroup and its parents. 1 (the +// enforce_exec behaviour) checks the task's own cgroup only. +#ifndef BINDINGS_CGROUP_DEPTH +#define BINDINGS_CGROUP_DEPTH 1 +#endif + +struct binding_value { + __u32 slot; + __u32 reserved; +}; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, BINDINGS_MAX); + __type(key, __u64); + __type(value, struct binding_value); +} bindings SEC(".maps"); + +// binding_lookup returns true when the current task's cgroup (or a parent up +// to BINDINGS_CGROUP_DEPTH) is bound, with the bound cgroup id and its slot. +static __always_inline bool binding_lookup(__u64 *cgroup, __u32 *slot) +{ + struct task_struct *task = bpf_get_current_task_btf(); + __u64 id = bpf_get_current_cgroup_id(); + struct binding_value *value = bpf_map_lookup_elem(&bindings, &id); + +#if BINDINGS_CGROUP_DEPTH > 1 + int level = BPF_CORE_READ(task, cgroups, dfl_cgrp, level); + + for (int i = 1; i < BINDINGS_CGROUP_DEPTH && !value; i++) { + if (level - i < 1) + break; + id = bpf_get_current_ancestor_cgroup_id(level - i); + value = bpf_map_lookup_elem(&bindings, &id); + } +#endif + if (!value) + return false; + *cgroup = id; + *slot = value->slot; + return true; +} + +#endif /* __ENFORCEMENT_BINDINGS_H */ diff --git a/pkg/ebpf/gadgets/common/enforcement/arm_linux.go b/pkg/ebpf/gadgets/common/enforcement/arm_linux.go new file mode 100644 index 0000000000..fea9596044 --- /dev/null +++ b/pkg/ebpf/gadgets/common/enforcement/arm_linux.go @@ -0,0 +1,120 @@ +// Copyright 2026 The Kubescape Authors +// SPDX-License-Identifier: Apache-2.0 + +package enforcement + +import ( + "bufio" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strconv" + + "golang.org/x/sys/unix" + + "github.com/inspektor-gadget/inspektor-gadget/pkg/utils/host" +) + +// pidfdThread is PIDFD_THREAD (Linux 6.9+): a pidfd for a single thread. +const pidfdThread = unix.O_EXCL + +// ArmExisting arms the tasks that already run in a bound container cgroup and +// its child cgroups. The BPF program arms tasks only at exec, so without this a +// container that started before the gadget would not be judged until its next +// exec. +// +// Tasks in the host mount namespace (a runtime helper such as runc init for +// exec, caught between entering the cgroup and joining the container) are +// skipped, as in the BPF arming rule. On kernels without PIDFD_THREAD, only +// thread group leaders are armed; the BPF rule treats their threads as armed. +// +// A task can fork between the read of cgroup.threads and its own arming; its +// child is then not armed by inheritance. So the walk repeats until a pass +// arms no new task. It returns ErrContainerGone when the cgroup no longer +// exists. +func ArmExisting(armed Map, cgroupPath string) error { + hostMntns, err := mntnsOf("1") + if err != nil { + return err + } + root := filepath.Join(host.HostRoot, "sys/fs/cgroup", cgroupPath) + seen := map[int]bool{} + for pass := 0; pass < maxArmingPasses; pass++ { + before := len(seen) + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + if path == root { + if errors.Is(err, fs.ErrNotExist) { + return ErrContainerGone + } + return fmt.Errorf("listing container tasks: %w", err) + } + return nil // a child cgroup removed during the walk + } + if !d.IsDir() { + return nil + } + return armThreads(armed, filepath.Join(path, "cgroup.threads"), hostMntns, seen) + }) + if err != nil { + return err + } + if len(seen) == before { + return nil + } + } + return fmt.Errorf("container tasks still changing after %d arming passes", maxArmingPasses) +} + +// ErrContainerGone means the container's cgroup no longer exists. +var ErrContainerGone = errors.New("container cgroup is gone") + +const maxArmingPasses = 8 + +func armThreads(armed Map, threads string, hostMntns uint64, seen map[int]bool) error { + f, err := os.Open(threads) + if err != nil { + return nil // the cgroup was removed + } + defer f.Close() + scanner := bufio.NewScanner(f) + for scanner.Scan() { + tid, err := strconv.Atoi(scanner.Text()) + if err != nil || seen[tid] { + continue + } + seen[tid] = true + if mntns, err := mntnsOf(scanner.Text()); err != nil || mntns == hostMntns { + continue + } + fd, err := unix.PidfdOpen(tid, pidfdThread) + if errors.Is(err, unix.EINVAL) { + fd, err = unix.PidfdOpen(tid, 0) + } + if err != nil { + // The task exited, or it is a non-leader thread on an older kernel. + continue + } + err = armed.Put(uint32(fd), uint32(1)) + unix.Close(fd) + if err != nil && !errors.Is(err, unix.ESRCH) { + return fmt.Errorf("arming task %d: %w", tid, err) + } + } + return scanner.Err() +} + +// RuntimeMountNamespace is the mount namespace of PID 1 as this process sees +// it: the namespace the container runtime runs in. Pass it to the gadget's +// host_mntns_id so the BPF arming rule and ArmExisting agree. +func RuntimeMountNamespace() (uint64, error) { return mntnsOf("1") } + +func mntnsOf(pid string) (uint64, error) { + var st unix.Stat_t + if err := unix.Stat(filepath.Join(host.HostProcFs, pid, "ns/mnt"), &st); err != nil { + return 0, err + } + return st.Ino, nil +} diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings.go b/pkg/ebpf/gadgets/common/enforcement/bindings.go new file mode 100644 index 0000000000..bccc68e614 --- /dev/null +++ b/pkg/ebpf/gadgets/common/enforcement/bindings.go @@ -0,0 +1,311 @@ +// Copyright 2026 The Kubescape Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package enforcement holds the native owner pieces shared by the enforcement +// gadgets. Bindings is the container binding controller of one node-wide gadget +// instance, after the enforce_exec simple engine in the Inspektor Gadget fork: +// the section's policy is written and frozen before any binding, container +// membership changes live, and a policy content change starts a replacement +// instance. It uses the enforce_exec identity type and `bindings` key and value +// size; the value holds the container's policy slot. +package enforcement + +import ( + "errors" + "fmt" + "strings" + "sync" + + "github.com/cilium/ebpf" + + execpolicy "github.com/inspektor-gadget/inspektor-gadget/gadgets/enforce_exec/policy" + execruntime "github.com/inspektor-gadget/inspektor-gadget/gadgets/enforce_exec/runtime" + "github.com/inspektor-gadget/inspektor-gadget/pkg/operators" +) + +// MaxBindings matches BINDINGS_MAX in ../bindings.h. +const MaxBindings = 1024 + +// ContainerIdentity is the enforce_exec identity, so one owner binds every gadget. +type ContainerIdentity = execruntime.ContainerIdentity + +// Binding places one container in one policy slot. +type Binding struct { + Container ContainerIdentity + Slot uint32 +} + +// bindingValue matches struct binding_value in ../bindings.h. +type bindingValue struct { + Slot uint32 + Reserved uint32 +} + +// Map is the part of *ebpf.Map the controller uses. +type Map interface { + Put(key, value interface{}) error + Delete(key interface{}) error + Type() ebpf.MapType + KeySize() uint32 + ValueSize() uint32 + MaxEntries() uint32 + Flags() uint32 + Freeze() error +} + +// Config describes one gadget instance. +type Config struct { + // Name of the native operator instance. + Name string + // Generation identifies the policy of this instance (nonzero). + Generation uint64 + InitialBindings []Binding + // Slots is the number of policy slots in the policy map. + Slots uint32 + // PolicyMap is the name of the section's policy map. + PolicyMap string + // WritePolicy checks the policy map layout and writes the entries. The + // controller freezes the map afterwards, before any binding. + WritePolicy func(Map) error + // ArmMap names the arming task-storage map. When set, binding a container + // also arms the tasks already running in it (see ../arming.h). + ArmMap string +} + +type Status struct { + Revision uint64 + Bound int + Ready bool + // Error is a startup or stop error: the instance does not protect. + Error error + // LastError is the result of the last live Bind or Unbind. It concerns + // that call only and does not affect Ready. + LastError error +} + +// Bindings serializes all writes. Map handles are borrowed from the eBPF operator. +type Bindings struct { + mu sync.Mutex + config Config + desired map[execpolicy.Container]Binding + active map[uint64]Binding + members, armed Map + ready, stopped bool + startupComplete bool + startupError error + lastError error + // armExisting is replaced in tests. + armExisting func(Map, string) error +} + +func New(config Config) (*Bindings, error) { + if config.Name == "" || config.Generation == 0 || config.PolicyMap == "" || config.WritePolicy == nil { + return nil, fmt.Errorf("enforcement controller requires a name, a generation and a policy writer") + } + if len(config.InitialBindings) > MaxBindings { + return nil, fmt.Errorf("too many initial bindings") + } + b := &Bindings{config: config, desired: map[execpolicy.Container]Binding{}, active: map[uint64]Binding{}, armExisting: ArmExisting} + for _, c := range config.InitialBindings { + if err := b.validate(c); err != nil { + return nil, err + } + if _, exists := b.desired[selector(c.Container)]; exists { + return nil, fmt.Errorf("duplicate initial container identity") + } + b.desired[selector(c.Container)] = c + } + return b, nil +} + +func (b *Bindings) Name() string { return b.config.Name } + +func (b *Bindings) Status() Status { + b.mu.Lock() + defer b.mu.Unlock() + return Status{Revision: b.config.Generation, Bound: len(b.active), Ready: b.ready && b.startupComplete && !b.stopped, Error: b.startupError, LastError: b.lastError} +} + +// Bind synchronously places one exact container in a slot. Binding a bound +// container to another slot moves it in one map write. +func (b *Bindings) Bind(c Binding) error { + b.mu.Lock() + defer b.mu.Unlock() + if !b.ready || b.stopped { + return b.recordError(fmt.Errorf("binding controller is not running")) + } + return b.recordError(b.bind(c)) +} + +// Unbind ignores stale removal notifications, which cannot remove a newer identity. +func (b *Bindings) Unbind(c ContainerIdentity) error { + b.mu.Lock() + defer b.mu.Unlock() + if !b.ready || b.stopped { + return b.recordError(fmt.Errorf("binding controller is not running")) + } + return b.recordError(b.unbind(c)) +} + +// Ready is the final startup boundary. +func (b *Bindings) Ready(operators.GadgetContext) error { + b.mu.Lock() + defer b.mu.Unlock() + if !b.ready || b.stopped { + return fmt.Errorf("binding controller is not running") + } + if b.startupError != nil { + return b.startupError + } + b.startupComplete = true + return nil +} + +// recordError is called with mu held. Startup errors stay sticky. +func (b *Bindings) recordError(err error) error { + b.lastError = err + if err != nil && !b.startupComplete && b.startupError == nil { + b.startupError = err + } + return err +} + +func selector(c ContainerIdentity) execpolicy.Container { + return execpolicy.Container{Runtime: c.Runtime, ID: c.ID} +} + +func (b *Bindings) validate(c Binding) error { + if c.Slot >= b.config.Slots { + return fmt.Errorf("policy slot %d out of range", c.Slot) + } + return validate(c.Container) +} + +func validate(c ContainerIdentity) error { + if strings.TrimSpace(c.Runtime) == "" || strings.TrimSpace(c.ID) == "" || c.CgroupID == 0 || c.CgroupPath == "/" || !strings.HasPrefix(c.CgroupPath, "/") { + return fmt.Errorf("binding requires runtime, full container ID and non-root cgroup v2 identity") + } + return nil +} + +func (b *Bindings) bind(c Binding) error { + if err := b.validate(c); err != nil { + return err + } + id := c.Container + previous, bound := b.active[id.CgroupID] + if bound && previous.Container != id { + return fmt.Errorf("cgroup identity conflicts with active binding") + } + if bound && previous.Slot == c.Slot { + return nil + } + if !bound { + for _, other := range b.active { + if selector(other.Container) == selector(id) { + return fmt.Errorf("container identity already bound to a different cgroup") + } + } + if len(b.active) >= MaxBindings { + return fmt.Errorf("container binding capacity exceeded") + } + } + if err := b.members.Put(id.CgroupID, bindingValue{Slot: c.Slot}); err != nil { + return fmt.Errorf("writing container binding: %w", err) + } + if !bound && b.armed != nil { + if err := b.armExisting(b.armed, id.CgroupPath); err != nil { + // A container that is gone needs no protection; a startup + // replay skips it (see start). + if derr := b.members.Delete(id.CgroupID); derr != nil { + err = errors.Join(err, derr) + } + return err + } + } + b.active[id.CgroupID] = c + return nil +} + +func (b *Bindings) unbind(c ContainerIdentity) error { + if b.desired[selector(c)].Container == c { + delete(b.desired, selector(c)) + } + if previous, ok := b.active[c.CgroupID]; !ok || previous.Container != c { + return nil + } + if err := b.members.Delete(c.CgroupID); err != nil && !errors.Is(err, ebpf.ErrKeyNotExist) { + return fmt.Errorf("removing container binding: %w", err) + } + delete(b.active, c.CgroupID) + return nil +} + +func (b *Bindings) Start(gc operators.GadgetContext) error { + getMap := func(name string) (Map, error) { + value, _ := gc.GetVar(operators.MapPrefix + name) + m, ok := value.(*ebpf.Map) + if !ok || m == nil { + return nil, fmt.Errorf("missing BPF map %s", name) + } + return m, nil + } + policy, err := getMap(b.config.PolicyMap) + if err != nil { + return err + } + members, err := getMap("bindings") + if err != nil { + return err + } + var armed Map + if b.config.ArmMap != "" { + if armed, err = getMap(b.config.ArmMap); err != nil { + return err + } + if armed.Type() != ebpf.TaskStorage { + return fmt.Errorf("invalid arming map layout") + } + } + return b.start(policy, members, armed) +} + +// start writes and freezes the policy before any binding is written. +func (b *Bindings) start(policy, members, armed Map) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.stopped { + return fmt.Errorf("binding controller already stopped") + } + if b.ready { + return fmt.Errorf("binding controller already started") + } + if members == nil || members.Type() != ebpf.Hash || members.KeySize() != 8 || members.ValueSize() != 8 || members.MaxEntries() != MaxBindings { + return fmt.Errorf("invalid binding map layout") + } + if err := b.config.WritePolicy(policy); err != nil { + return err + } + if err := policy.Freeze(); err != nil { + return fmt.Errorf("freezing policy: %w", err) + } + b.members, b.armed = members, armed + for _, c := range b.desired { + if err := b.bind(c); err != nil && !errors.Is(err, ErrContainerGone) { + return b.recordError(err) + } + } + b.desired = nil + b.ready = true + return nil +} + +func (b *Bindings) Stop(operators.GadgetContext) error { + b.mu.Lock() + defer b.mu.Unlock() + b.stopped = true + b.ready = false + return nil +} + +func (b *Bindings) Close(gc operators.GadgetContext) error { return b.Stop(gc) } diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go new file mode 100644 index 0000000000..29df707e10 --- /dev/null +++ b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go @@ -0,0 +1,154 @@ +// Copyright 2026 The Kubescape Authors +// SPDX-License-Identifier: Apache-2.0 + +package enforcement + +import ( + "errors" + "strings" + "testing" + + "github.com/cilium/ebpf" + "golang.org/x/sys/unix" +) + +type fakeMap struct { + typ ebpf.MapType + key, value, max, flag uint32 + data map[interface{}]interface{} + frozen bool + log *[]string + name string +} + +func (m *fakeMap) Put(k, v interface{}) error { + if m.frozen { + return unix.EPERM + } + *m.log = append(*m.log, "put "+m.name) + m.data[k] = v + return nil +} + +func (m *fakeMap) Delete(k interface{}) error { + if _, ok := m.data[k]; !ok { + return ebpf.ErrKeyNotExist + } + delete(m.data, k) + return nil +} +func (m *fakeMap) Type() ebpf.MapType { return m.typ } +func (m *fakeMap) KeySize() uint32 { return m.key } +func (m *fakeMap) ValueSize() uint32 { return m.value } +func (m *fakeMap) MaxEntries() uint32 { return m.max } +func (m *fakeMap) Flags() uint32 { return m.flag } +func (m *fakeMap) Freeze() error { + *m.log = append(*m.log, "freeze "+m.name) + m.frozen = true + return nil +} + +var identity = ContainerIdentity{Runtime: "docker", ID: "abc", CgroupID: 42, CgroupPath: "/system.slice/docker-abc.scope"} + +func newFixture(t *testing.T, write func(Map) error) (*Bindings, *fakeMap, *fakeMap, *[]string) { + t.Helper() + log := &[]string{} + policy := &fakeMap{typ: ebpf.Hash, data: map[interface{}]interface{}{}, log: log, name: "policy"} + members := &fakeMap{typ: ebpf.Hash, key: 8, value: 8, max: MaxBindings, data: map[interface{}]interface{}{}, log: log, name: "bindings"} + b, err := New(Config{Name: "test", Generation: 5, Slots: 2, PolicyMap: "policy", WritePolicy: write, InitialBindings: []Binding{{Container: identity}}}) + if err != nil { + t.Fatal(err) + } + return b, policy, members, log +} + +func TestFreezeBeforeBindAndLifecycle(t *testing.T) { + b, policy, members, log := newFixture(t, func(m Map) error { return m.Put(uint32(0), uint32(1)) }) + armed := &fakeMap{typ: ebpf.TaskStorage} + var armedPaths []string + b.armExisting = func(_ Map, path string) error { armedPaths = append(armedPaths, path); return nil } + if err := b.start(policy, members, armed); err != nil { + t.Fatal(err) + } + if err := b.Ready(nil); err != nil { + t.Fatal(err) + } + if got := strings.Join(*log, ","); got != "put policy,freeze policy,put bindings" { + t.Fatalf("order %s", got) + } + if len(armedPaths) != 1 || armedPaths[0] != identity.CgroupPath { + t.Fatalf("armed %v", armedPaths) + } + st := b.Status() + if !st.Ready || st.Bound != 1 || st.Revision != 5 { + t.Fatalf("status %+v", st) + } + if err := b.Bind(Binding{Container: identity}); err != nil { + t.Fatal(err) + } + // Moving to another slot is one map write and no second arming. + if err := b.Bind(Binding{Container: identity, Slot: 1}); err != nil || members.data[uint64(42)] != (bindingValue{Slot: 1}) || len(armedPaths) != 1 { + t.Fatalf("slot move: %v %v armed=%v", err, members.data, armedPaths) + } + if err := b.Bind(Binding{Container: identity, Slot: 2}); err == nil { + t.Fatal("slot out of range must fail") + } + other := identity + other.ID = "def" + if err := b.Bind(Binding{Container: other}); err == nil { + t.Fatal("conflicting cgroup identity must fail") + } + if err := b.Unbind(other); err != nil || len(members.data) != 1 { + t.Fatalf("stale unbind: %v %v", err, members.data) + } + if err := b.Unbind(identity); err != nil || len(members.data) != 0 { + t.Fatalf("unbind: %v %v", err, members.data) + } +} + +func TestFailedArmingRemovesBinding(t *testing.T) { + b, policy, members, _ := newFixture(t, func(Map) error { return nil }) + b.armExisting = func(Map, string) error { return errors.New("no cgroup") } + if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}); err == nil { + t.Fatal("arming failure must fail startup") + } + if len(members.data) != 0 { + t.Fatalf("binding must be removed: %v", members.data) + } +} + +func TestRejectsWrongBindingLayoutAndPolicyErrors(t *testing.T) { + b, policy, members, log := newFixture(t, func(Map) error { return nil }) + members.max = 8 + if err := b.start(policy, members, nil); err == nil || len(*log) != 0 { + t.Fatalf("wrong binding layout must fail before writes: %v %v", err, *log) + } + b, policy, members, log = newFixture(t, func(Map) error { return errors.New("bad layout") }) + if err := b.start(policy, members, nil); err == nil || len(*log) != 0 { + t.Fatalf("policy writer error must stop startup: %v %v", err, *log) + } +} + +func TestNewValidates(t *testing.T) { + if _, err := New(Config{Name: "x", PolicyMap: "p", WritePolicy: func(Map) error { return nil }}); err == nil { + t.Fatal("zero generation must fail") + } +} + +func TestStartSkipsGoneContainer(t *testing.T) { + b, policy, members, _ := newFixture(t, func(Map) error { return nil }) + b.armExisting = func(Map, string) error { return ErrContainerGone } + if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}); err != nil { + t.Fatalf("a gone container must not fail startup: %v", err) + } + if err := b.Ready(nil); err != nil || len(members.data) != 0 { + t.Fatalf("ready=%v bindings=%v", err, members.data) + } + // A failed live call does not make the instance unready. + if err := b.Bind(Binding{Container: identity}); err == nil { + t.Fatal("bind of a gone container must report it") + } + if st := b.Status(); !st.Ready || st.Error != nil || st.LastError == nil { + t.Fatalf("status %+v", st) + } +} diff --git a/pkg/ebpf/gadgets/common/policy.h b/pkg/ebpf/gadgets/common/policy.h new file mode 100644 index 0000000000..ea663ac4c3 --- /dev/null +++ b/pkg/ebpf/gadgets/common/policy.h @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright 2026 The Kubescape Authors + +// Policy resolution shared by the enforcement gadgets. +// +// The program is fixed; the native owner writes the section's policy map and +// freezes it before any binding is written. A rule change starts a replacement +// gadget instance, and the generation identifies the instance's policy. Modes +// use the enforce_exec encoding. +// +// Each policy value holds a rank. The compiler folds every rule that hits one +// key into the most limiting rank, and the program takes the maximum over every +// key a hook touches, so resolution is: explicit deny > allow > section default, +// independent of rule order, with one mode per rule. Key layouts are per section. + +#ifndef __ENFORCEMENT_POLICY_H +#define __ENFORCEMENT_POLICY_H + +#include + +#define POLICY_MODE_OFF 0 +#define POLICY_MODE_AUDIT 1 +#define POLICY_MODE_ENFORCE 2 + +enum policy_rank : __u8 { + POLICY_RANK_NONE = 0, + POLICY_RANK_ALLOW = 1, + POLICY_RANK_DENY_AUDIT = 2, + POLICY_RANK_DENY_ENFORCE = 3, +}; + +struct policy_value { + __u8 rank; + __u8 pad[3]; + // Rule and origin ids index name tables kept by the native owner. 0 = none. + __u32 rule_id; + __u32 origin_id; + // Set only on the section's header entry. + __u32 mode; + __u64 generation; +}; + +// The result of combining every entry a hook touched. +struct policy_match { + __u8 rank; + __u8 is_default; + __u32 rule_id; + __u32 origin_id; +}; + +// policy_take keeps v in m when it is more limiting than what m holds. +static __always_inline void policy_take(struct policy_match *m, + const struct policy_value *v, + bool is_default) +{ + if (!v || v->rank <= m->rank) + return; + m->rank = v->rank; + m->is_default = is_default; + m->rule_id = v->rule_id; + m->origin_id = v->origin_id; +} + +static __always_inline void policy_merge(struct policy_match *dst, + const struct policy_match *src) +{ + if (src->rank > dst->rank) + *dst = *src; +} + +#endif /* __ENFORCEMENT_POLICY_H */ diff --git a/pkg/ebpf/gadgets/common/verdict.h b/pkg/ebpf/gadgets/common/verdict.h new file mode 100644 index 0000000000..ea88abdedc --- /dev/null +++ b/pkg/ebpf/gadgets/common/verdict.h @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright 2026 The Kubescape Authors + +// Verdict reporting shared by the enforcement gadgets. Names, order, limits and +// counter slots follow the enforce_exec gadget, so one consumer decodes both. +// Reporting is limited to 100 attempts per CPU per second; enforcement is never +// throttled. + +#ifndef __ENFORCEMENT_VERDICT_H +#define __ENFORCEMENT_VERDICT_H + +#include +#include + +#include "policy.h" + +// First fields of every verdict event, in enforce_exec order. mode: 1 audit, +// 2 enforce. +#define VERDICT_BASE_FIELDS \ + __u64 generation; \ + __u64 cgroup; \ + __u32 deny; \ + __u32 mode + +#define VERDICT_FILL_BASE(e, m, gen, cgroup_id) \ + do { \ + (e)->generation = (gen); \ + (e)->cgroup = (cgroup_id); \ + (e)->deny = (m)->rank >= POLICY_RANK_DENY_AUDIT; \ + (e)->mode = (m)->rank == POLICY_RANK_DENY_ENFORCE ? \ + POLICY_MODE_ENFORCE : \ + POLICY_MODE_AUDIT; \ + } while (0) + +// Counter slots, as in enforce_exec. +enum verdict_counter { + VERDICT_COUNT_DENIED = 0, // deny decisions, audit and enforce + VERDICT_COUNT_SUBMITTED = 1, + VERDICT_COUNT_DROPPED = 2, + VERDICT_COUNT_ALLOWED = 3, + VERDICT_COUNT_SECTION_ERROR = 4, + VERDICT_COUNT_RATE_LIMITED = 5, + VERDICT_COUNT_MAX = 6, +}; + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, VERDICT_COUNT_MAX); + __type(key, __u32); + __type(value, __u64); +} counters SEC(".maps"); + +static __always_inline void verdict_count(__u32 key) +{ + __u64 *value = bpf_map_lookup_elem(&counters, &key); + + if (value) + (*value)++; +} + +#define VERDICT_REPORTS_PER_SECOND 100 + +struct report_window { + __u64 second; + __u32 attempts; +}; + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct report_window); +} report_windows SEC(".maps"); + +static __always_inline bool verdict_report_allowed(void) +{ + __u32 zero = 0; + struct report_window *window = + bpf_map_lookup_elem(&report_windows, &zero); + __u64 second; + + if (!window) + return false; + second = bpf_ktime_get_ns() / 1000000000; + if (window->second != second) { + window->second = second; + window->attempts = 0; + } + if (window->attempts >= VERDICT_REPORTS_PER_SECOND) + return false; + window->attempts++; + return true; +} + +#endif /* __ENFORCEMENT_VERDICT_H */ diff --git a/pkg/ebpf/gadgets/enforce_privilege/Makefile b/pkg/ebpf/gadgets/enforce_privilege/Makefile new file mode 100644 index 0000000000..d7290e9224 --- /dev/null +++ b/pkg/ebpf/gadgets/enforce_privilege/Makefile @@ -0,0 +1,8 @@ +IMAGE?=enforce_privilege +TAG?=latest +# The gadget includes ../common headers. IG_SOURCE_PATH makes `ig image build` +# mount the repository root (instead of this folder only) so they resolve. +REPO_ROOT:=$(abspath $(CURDIR)/../../../..) + +build: + sudo TMPDIR=$$TMPDIR IG_SOURCE_PATH=$(REPO_ROOT) ig image build -t $(IMAGE):$(TAG) . diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md new file mode 100644 index 0000000000..e7eba05e86 --- /dev/null +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -0,0 +1,108 @@ +# enforce_privilege + +Container privilege enforcement with BPF LSM, no seccomp. It enforces the +`privilege` section of a sandbox policy: namespace creation and joining, mounts, +`pivot_root`/`chroot`, capabilities, ptrace, kernel modules, `bpf`, +`perf_event_open`, keys, `io_uring_setup`, `userfaultfd`, `mknod`, +`setuid`/`setgid` and `capset`. + +The gadget is built on the [common enforcement library](../common/README.md). +One instance runs per node: + +- A **slot** is one combination of privilege policies; containers with the same + combination share it. Limits per node: 1024 containers, 64 slots, 512 policy + map entries per slot, 16384 in total. +- Binding or unbinding a container is a live write to the `bindings` map. +- A combination the running instance does not have (a new or edited policy) + replaces the node instance: the candidate starts with every slot in use, is + ready with every container bound, and then the old instance stops. Both + enforce during the overlap. Unused slots are dropped at the next replacement. +- An owner can create a slot before a container starts (for example when its + pod is scheduled to the node), so the container binds with a live write. +- Several policies on one container merge as the SandboxPolicy design says: the + union of rules (explicit deny > allow > default, each rule with its own name, + origin and mode); a default is deny if any policy denies, with the strictest + mode among the policies that agree. One addition to the design: a monitoring + (audit) deny rule never weakens another policy's enforce deny, including its + enforce default deny. An explicit allow still beats another policy's default + deny. + +`policy/` compiles the `privilege` section of one or more policies; +`runtime/` is the native owner a controller drives. + +Requires cgroup v2, kernel BTF and BPF LSM enabled (`bpf` in +`/sys/kernel/security/lsm`). Linux 6.1 or later (the `userns_create` hook; +`io_uring_setup` reaches `inode_init_security_anon` from 5.16). +Build the image with `make build` (it sets `IG_SOURCE_PATH`, see the library +README). + +## Policy + +The `privilege` section of a SandboxPolicy, for example: + +```yaml +spec: + mode: enforce # audit | enforce | off + privilege: + default: allow # applies to syscall and mountFsType targets + rules: + - name: no-new-namespaces + target: {syscall: [unshare, clone, clone3, setns], flags: [CLONE_NEWUSER, CLONE_NEWNS]} + action: deny + origin: catalog/G1 + - name: no-proc-mount + target: {mountFsType: [proc, sysfs]} + action: deny + - name: audit-sys-admin + target: {capability: [SYS_ADMIN]} + action: deny + mode: audit +``` + +- A rule targets exactly one kind: `syscall` (optionally with `flags` for + `unshare`, `setns`, `clone`, `clone3`), `capability` (short names such as + `SYS_ADMIN`) or `mountFsType`. +- Resolution: an explicit deny beats an allow, an allow beats the default, + independent of rule order. Each rule has its own mode. +- Capability checks have no default; only explicit capability rules apply. +- A `default: deny` also denies `setuid`/`setgid` (package managers drop + privileges), implicit kernel module loads (judged as `init_module`, for + example an `iptables` match module) and new superblocks of any filesystem + type. Allow what the workload needs. +- `actor`, `except` and `pathClass` are not supported yet. + +## How it decides + +- **Arming.** Only armed tasks are judged. A task is armed when it executes a + program outside the container runtime's mount namespace; children inherit the + mark. So runc's own setup (namespaces, mounts, `pivot_root`, `setns` for + `exec`) is never blocked, and `docker run` / `docker exec` keep working. + When a container is bound, the owner also arms the tasks that already run in + it (pidfd writes to `armed_tasks`). The runtime mount namespace is the + `host-mntns-id` gadget parameter; the owner sets it to the mount namespace of + PID 1 as it sees it (in kind, the node container's). When it is 0, the + program uses the global init task's. +- **Scope.** A bound cgroup also covers up to 31 levels of child cgroups. +- **Capabilities.** A `capable()` call is judged only when the task uses its own + credentials (overlayfs copy-up uses the mounter's) and the call is not a + no-audit probe. +- **Syscalls without their own hook** (`unshare` without a user namespace, + `setns`, `open_tree`, `fsopen`, `fsmount`, `fspick`, `mount_setattr`) are + judged at their `CAP_SYS_ADMIN` check by syscall number. For a 32-bit + (compat) task the number is not mapped, so these checks fail closed: every + candidate syscall rule applies. +- **Deny** returns `-EPERM` from the LSM hook, before the side effect. + +## Maps and events + +| Map | Content | +|---|---| +| `policy` | frozen hash: `{slot, kind, id, qual, name}` → `{rank, rule_id, origin_id}`; each slot's META entry holds generation and mode (`privilege_policy.h`) | +| `bindings`, `armed_tasks`, `counters` | from the common library | + +The `verdict` datasource shares `generation`, `cgroup`, `deny`, `mode` and +`host_pid` with enforce_exec and adds `rule_id`, `origin_id`, `is_default`, +`comm`, `hook`, `syscall`, `flags`, `capability`, `fstype` and `slot`. Rule and origin +names are in the owner's compiled policy (`RuleNames`, `Origins`). Reporting is +limited to 100 events per CPU per second; enforcement is never throttled. + diff --git a/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml b/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml new file mode 100644 index 0000000000..9024f4ffff --- /dev/null +++ b/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml @@ -0,0 +1,24 @@ +name: enforce_privilege +behavior: mutator +description: "Enforce SandboxPolicy privilege rules (capabilities, namespaces, mounts, ptrace, modules) for a live set of containers (BPF LSM)" +annotations: + enable-containers-datasource: "true" + enforce-privilege: "true" +datasources: + verdict: + fields: + hook_raw: + annotations: + columns.hidden: true + syscall_raw: + annotations: + columns.hidden: true + containers: + annotations: + cli.default-output-mode: none +params: + ebpf: + host_mntns_id: + key: host-mntns-id + defaultValue: "0" + description: mount namespace inode of the container runtime; 0 uses the global init task's diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/api.go b/pkg/ebpf/gadgets/enforce_privilege/policy/api.go new file mode 100644 index 0000000000..0b6294b907 --- /dev/null +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/api.go @@ -0,0 +1,24 @@ +package privpolicy + +import ( + "fmt" + + api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" +) + +// FromAPI converts a SandboxPolicy privilege section into a validated Document. +// Actor and path predicates are not supported yet and are rejected. +func FromAPI(name string, mode api.PolicyMode, section api.PrivilegeSection) (Document, error) { + doc := Document{Name: name, Version: 1, Policy: Policy{Mode: string(mode), Privilege: Section{Default: string(section.Default)}}} + for _, r := range section.Rules { + t := r.Target + if r.Actor != nil || len(t.Path) > 0 || len(t.PathClass) > 0 || t.FromImage != nil || len(t.FSType) > 0 { + return doc, fmt.Errorf("rule %q: actor and path targets are unsupported in the privilege section", r.Name) + } + doc.Policy.Privilege.Rules = append(doc.Policy.Privilege.Rules, Rule{ + Name: r.Name, Action: string(r.Action), Mode: string(r.Mode), Origin: r.Origin, + Target: Target{Syscall: t.Syscall, Flags: t.Flags, Capability: t.Capability, MountFSType: t.MountFSType}, + }) + } + return doc, doc.Validate() +} diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go new file mode 100644 index 0000000000..ff7d7b121e --- /dev/null +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go @@ -0,0 +1,332 @@ +// Copyright 2026 The Kubescape Authors +// SPDX-License-Identifier: Apache-2.0 + +package privpolicy + +import ( + "fmt" + "sort" +) + +// Map ABI of ../privilege_policy.h and ../../common/policy.h. Keep in sync. +const ( + // Per node instance: slots, entries per slot and entries in total. + KernelMaxSlots = 64 + KernelMaxSlotEntries = 512 + KernelMaxEntries = 16384 + KernelNameLen = 16 + KernelKeySize = 40 + KernelValueSize = 24 + + KindMeta = 0 + KindDefault = 1 + KindSyscall = 2 + KindCapability = 3 + KindMountFSType = 4 + + RankNone = 0 + RankAllow = 1 + RankDenyAudit = 2 + RankDenyEnforce = 3 + KernelModeOff = 0 + KernelModeAudit = 1 + KernelModeEnforce = 2 +) + +// Syscalls maps policy names to the program's priv_syscall ids. +var Syscalls = map[string]uint32{ + "unshare": 1, "setns": 2, "clone": 3, "clone3": 4, "mount": 5, "umount2": 6, + "pivot_root": 7, "chroot": 8, "open_tree": 9, "move_mount": 10, "fsopen": 11, + "fsmount": 12, "fspick": 13, "mount_setattr": 14, "ptrace": 15, + "process_vm_readv": 16, "process_vm_writev": 17, "init_module": 18, + "finit_module": 19, "bpf": 20, "perf_event_open": 21, "kexec_load": 22, + "keyctl": 23, "userfaultfd": 24, "io_uring_setup": 25, "mknod": 26, + "setuid": 27, "setgid": 28, "capset": 29, +} + +var NamespaceSyscalls = map[string]bool{"unshare": true, "setns": true, "clone": true, "clone3": true} + +var namespaceSyscallIDs = map[uint32]bool{1: true, 2: true, 3: true, 4: true} + +var NamespaceFlags = map[string]uint64{ + "CLONE_NEWNS": 0x00020000, "CLONE_NEWCGROUP": 0x02000000, "CLONE_NEWUTS": 0x04000000, + "CLONE_NEWIPC": 0x08000000, "CLONE_NEWUSER": 0x10000000, "CLONE_NEWPID": 0x20000000, + "CLONE_NEWNET": 0x40000000, "CLONE_NEWTIME": 0x00000080, +} + +var Capabilities = map[string]uint32{ + "CHOWN": 0, "DAC_OVERRIDE": 1, "DAC_READ_SEARCH": 2, "FOWNER": 3, "FSETID": 4, + "KILL": 5, "SETGID": 6, "SETUID": 7, "SETPCAP": 8, "LINUX_IMMUTABLE": 9, + "NET_BIND_SERVICE": 10, "NET_BROADCAST": 11, "NET_ADMIN": 12, "NET_RAW": 13, + "IPC_LOCK": 14, "IPC_OWNER": 15, "SYS_MODULE": 16, "SYS_RAWIO": 17, + "SYS_CHROOT": 18, "SYS_PTRACE": 19, "SYS_PACCT": 20, "SYS_ADMIN": 21, + "SYS_BOOT": 22, "SYS_NICE": 23, "SYS_RESOURCE": 24, "SYS_TIME": 25, + "SYS_TTY_CONFIG": 26, "MKNOD": 27, "LEASE": 28, "AUDIT_WRITE": 29, + "AUDIT_CONTROL": 30, "SETFCAP": 31, "MAC_OVERRIDE": 32, "MAC_ADMIN": 33, + "SYSLOG": 34, "WAKE_ALARM": 35, "BLOCK_SUSPEND": 36, "AUDIT_READ": 37, + "PERFMON": 38, "BPF": 39, "CHECKPOINT_RESTORE": 40, +} + +// KernelKey and KernelValue match struct policy_key and struct policy_value. +type KernelKey struct { + Slot uint32 + Kind uint32 + ID uint32 + _ uint32 + Qual uint64 + Name [KernelNameLen]byte +} + +type KernelValue struct { + Rank uint8 + _ [3]uint8 + RuleID uint32 + OriginID uint32 + Mode uint32 + Generation uint64 +} + +type KernelEntry struct { + Key KernelKey + Value KernelValue +} + +// Compiled is the frozen map content plus the id tables events refer to. +// RuleNames[0] and Origins[0] are empty: id 0 means none. +type Compiled struct { + Generation uint64 + // Slots is the number of policy slots (combinations of policies). + Slots uint32 + Entries []KernelEntry + RuleNames []string + Origins []string +} + +func rank(action, mode string) uint8 { + switch { + case action == "allow": + return RankAllow + case mode == "enforce": + return RankDenyEnforce + default: + return RankDenyAudit + } +} + +// CompileKernel compiles one policy into slot 0. +func CompileKernel(d Document, generation uint64) (Compiled, error) { + return CompileSlots([][]Document{{d}}, generation) +} + +// CompileSlots compiles one node instance. Each slot holds one combination of +// policies, merged as the CRD design says for several policies matching one pod: +// - rules are the union, each with its own name, origin and mode; the compiler +// folds every rule on one key into the most limiting rank, so an explicit +// deny beats an allow and an allow beats a default; +// - a default is deny if any policy's default is deny, else allow; its mode is +// the strictest among the policies whose default equals the merged one. +// +// The default covers the syscall and mountFsType kinds only. Capability checks +// have no default: only explicit capability rules apply, because capable() runs +// for many benign reasons. +func CompileSlots(slots [][]Document, generation uint64) (Compiled, error) { + if generation == 0 { + return Compiled{}, fmt.Errorf("generation must be nonzero") + } + if len(slots) > KernelMaxSlots { + return Compiled{}, fmt.Errorf("%d policy combinations exceed %d per node", len(slots), KernelMaxSlots) + } + c := Compiled{Generation: generation, RuleNames: []string{"", "default"}, Origins: []string{""}} + originIDs := map[string]uint32{} + originID := func(o string) uint32 { + if o == "" { + return 0 + } + if id, ok := originIDs[o]; ok { + return id + } + c.Origins = append(c.Origins, o) + originIDs[o] = uint32(len(c.Origins) - 1) + return originIDs[o] + } + for slot, docs := range slots { + if len(docs) == 0 { + return Compiled{}, fmt.Errorf("slot %d has no policy", slot) + } + entries, err := compileSlot(uint32(slot), docs, generation, &c, originID) + if err != nil { + return Compiled{}, err + } + if len(entries) > KernelMaxSlotEntries { + return Compiled{}, fmt.Errorf("policy combination expands to %d map entries (max %d)", len(entries), KernelMaxSlotEntries) + } + c.Entries = append(c.Entries, entries...) + } + c.Slots = uint32(len(slots)) + if len(c.Entries) > KernelMaxEntries { + return Compiled{}, fmt.Errorf("policies expand to %d map entries (max %d)", len(c.Entries), KernelMaxEntries) + } + return c, nil +} + +func compileSlot(slot uint32, docs []Document, generation uint64, c *Compiled, originID func(string) uint32) ([]KernelEntry, error) { + meta := KernelEntry{Key: KernelKey{Slot: slot, Kind: KindMeta}, Value: KernelValue{Generation: generation}} + var active []Document + for _, d := range docs { + if err := d.Validate(); err != nil { + return nil, err + } + switch d.Policy.Mode { + case "enforce": + meta.Value.Mode = KernelModeEnforce + case "audit": + if meta.Value.Mode == KernelModeOff { + meta.Value.Mode = KernelModeAudit + } + } + if d.Policy.Mode != "off" { + active = append(active, d) + } + } + if len(active) == 0 { + return []KernelEntry{meta}, nil + } + + folded := map[KernelKey]KernelValue{} + // Per rule key: whether any rule allows, and which policies have a rule. + allowed := map[KernelKey]bool{} + ruleOf := map[KernelKey]map[int]bool{} + doc := 0 + put := func(k KernelKey, v KernelValue) { + k.Slot = slot + if old, ok := folded[k]; !ok || v.Rank > old.Rank { + folded[k] = v + } + if k.Kind == KindDefault { + return + } + if v.Rank == RankAllow { + allowed[k] = true + } + if ruleOf[k] == nil { + ruleOf[k] = map[int]bool{} + } + ruleOf[k][doc] = true + } + + // Merged default: deny if any policy denies; the strictest mode among the + // policies that agree with the merged verdict. + merged := "allow" + for _, d := range active { + if d.Policy.Privilege.Default == "deny" { + merged = "deny" + } + } + mode := "audit" + for _, d := range active { + if d.Policy.Privilege.Default == merged && d.Policy.Mode == "enforce" { + mode = "enforce" + } + } + def := KernelValue{Rank: rank(merged, mode), RuleID: 1} + put(KernelKey{Kind: KindDefault, ID: KindSyscall}, def) + put(KernelKey{Kind: KindDefault, ID: KindMountFSType}, def) + + for i, d := range active { + doc = i + for _, r := range d.Policy.Privilege.Rules { + mode := r.Mode + if mode == "" { + mode = d.Policy.Mode + } + name := r.Name + if d.Name != "" { + name = d.Name + "/" + r.Name + } + c.RuleNames = append(c.RuleNames, name) + v := KernelValue{Rank: rank(r.Action, mode), RuleID: uint32(len(c.RuleNames) - 1), OriginID: originID(r.Origin)} + t := r.Target + for _, sc := range t.Syscall { + if len(t.Flags) == 0 { + put(KernelKey{Kind: KindSyscall, ID: Syscalls[sc]}, v) + } + for _, f := range t.Flags { + put(KernelKey{Kind: KindSyscall, ID: Syscalls[sc], Qual: NamespaceFlags[f]}, v) + } + } + for _, cp := range t.Capability { + put(KernelKey{Kind: KindCapability, ID: Capabilities[cp]}, v) + } + for _, fs := range t.MountFSType { + k := KernelKey{Kind: KindMountFSType} + copy(k.Name[:], fs) + put(k, v) + } + } + } + + // A monitoring (audit) deny rule must never weaken another policy's enforce + // deny, including its default deny: on a key where every matching rule + // denies, the default deny of each policy without a rule on that key still + // counts. An explicit allow still beats another policy's default deny. + // + // The program combines {syscall, 0} with every {syscall, flag} of a call, so + // namespace syscalls are handled per flag: a rule on {syscall, 0} also + // matches every flag, and only the per-flag keys are raised. + raise := func(k KernelKey, hasRule func(i int) bool) { + v := folded[k] + for i, d := range active { + if d.Policy.Privilege.Default != "deny" || hasRule(i) { + continue + } + if r := rank("deny", d.Policy.Mode); r > v.Rank { + v = KernelValue{Rank: r, RuleID: 1} + } + } + if v.Rank != RankNone { + folded[k] = v + } + } + for k := range folded { + if k.Kind == KindMountFSType || (k.Kind == KindSyscall && !namespaceSyscallIDs[k.ID]) { + if !allowed[k] { + raise(k, func(i int) bool { return ruleOf[k][i] }) + } + } + } + for id := range namespaceSyscallIDs { + anyFlag := KernelKey{Slot: slot, Kind: KindSyscall, ID: id} + for _, flag := range NamespaceFlags { + k := anyFlag + k.Qual = flag + denied := folded[anyFlag].Rank >= RankDenyAudit || folded[k].Rank >= RankDenyAudit + if !denied || allowed[anyFlag] || allowed[k] { + continue + } + raise(k, func(i int) bool { return ruleOf[anyFlag][i] || ruleOf[k][i] }) + } + } + + entries := []KernelEntry{meta} + keys := make([]KernelKey, 0, len(folded)) + for k := range folded { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + a, b := keys[i], keys[j] + if a.Kind != b.Kind { + return a.Kind < b.Kind + } + if a.ID != b.ID { + return a.ID < b.ID + } + if a.Qual != b.Qual { + return a.Qual < b.Qual + } + return string(a.Name[:]) < string(b.Name[:]) + }) + for _, k := range keys { + entries = append(entries, KernelEntry{Key: k, Value: folded[k]}) + } + return entries, nil +} diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go new file mode 100644 index 0000000000..a7ebc481f8 --- /dev/null +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go @@ -0,0 +1,249 @@ +// Copyright 2026 The Kubescape Authors +// SPDX-License-Identifier: Apache-2.0 + +package privpolicy + +import ( + "encoding/binary" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// parse decodes a test policy with unknown fields rejected, then validates it. +func parse(y string) (Document, error) { + var d Document + dec := yaml.NewDecoder(strings.NewReader(y)) + dec.KnownFields(true) + if err := dec.Decode(&d); err != nil { + return d, err + } + return d, d.Validate() +} + +func mustParse(t *testing.T, y string) Document { + t.Helper() + d, err := parse(y) + if err != nil { + t.Fatal(err) + } + return d +} + +func lookup(c Compiled, k KernelKey) (KernelValue, bool) { + for _, e := range c.Entries { + if e.Key == k { + return e.Value, true + } + } + return KernelValue{}, false +} + +func TestKernelABISizes(t *testing.T) { + if n := binary.Size(KernelKey{}); n != KernelKeySize { + t.Fatalf("key size %d", n) + } + if n := binary.Size(KernelValue{}); n != KernelValueSize { + t.Fatalf("value size %d", n) + } +} + +const sample = ` +version: 1 +policy: + mode: audit + privilege: + default: allow + rules: + - name: no-new-ns + target: {syscall: [unshare, clone3], flags: [CLONE_NEWNS, CLONE_NEWUSER]} + action: deny + mode: enforce + origin: catalog/G1 + - name: allow-unshare-ns + target: {syscall: [unshare], flags: [CLONE_NEWNS]} + action: allow + - name: audit-sysadmin + target: {capability: [SYS_ADMIN]} + action: deny + - name: no-proc + target: {mountFsType: [proc]} + action: deny + mode: enforce +` + +func TestCompileFoldsMostLimiting(t *testing.T) { + c, err := CompileKernel(mustParse(t, sample), 7) + if err != nil { + t.Fatal(err) + } + meta, _ := lookup(c, KernelKey{Kind: KindMeta}) + if meta.Generation != 7 || meta.Mode != KernelModeAudit { + t.Fatalf("meta %+v", meta) + } + // Deny+enforce beats the allow on the same key, independent of order. + v, ok := lookup(c, KernelKey{Kind: KindSyscall, ID: Syscalls["unshare"], Qual: NamespaceFlags["CLONE_NEWNS"]}) + if !ok || v.Rank != RankDenyEnforce || c.RuleNames[v.RuleID] != "no-new-ns" || c.Origins[v.OriginID] != "catalog/G1" { + t.Fatalf("unshare/NEWNS %+v ok=%v", v, ok) + } + // Rule mode inherits the policy mode. + v, _ = lookup(c, KernelKey{Kind: KindCapability, ID: Capabilities["SYS_ADMIN"]}) + if v.Rank != RankDenyAudit { + t.Fatalf("SYS_ADMIN rank %d", v.Rank) + } + var proc KernelKey + proc.Kind = KindMountFSType + copy(proc.Name[:], "proc") + if v, ok = lookup(c, proc); !ok || v.Rank != RankDenyEnforce { + t.Fatalf("proc %+v", v) + } + // The default covers syscall and mountFsType, never capability. + if v, ok = lookup(c, KernelKey{Kind: KindDefault, ID: KindSyscall}); !ok || v.Rank != RankAllow || c.RuleNames[v.RuleID] != "default" { + t.Fatalf("syscall default %+v", v) + } + if _, ok = lookup(c, KernelKey{Kind: KindDefault, ID: KindCapability}); ok { + t.Fatal("capability default must not be written") + } + if c.Entries[0].Key.Kind != KindMeta { + t.Fatal("META must be the first entry") + } +} + +func TestOffLoadsOnlyMeta(t *testing.T) { + c, err := CompileKernel(mustParse(t, "version: 1\npolicy: {mode: off, privilege: {default: deny}}\n"), 3) + if err != nil { + t.Fatal(err) + } + if len(c.Entries) != 1 || c.Entries[0].Value.Mode != KernelModeOff { + t.Fatalf("entries %+v", c.Entries) + } +} + +func TestValidationErrors(t *testing.T) { + base := "version: 1\npolicy:\n mode: enforce\n privilege:\n default: allow\n rules:\n" + for name, rule := range map[string]string{ + "two kinds": " - {name: r, target: {syscall: [mount], capability: [SYS_ADMIN]}, action: deny}\n", + "no kind": " - {name: r, target: {}, action: deny}\n", + "flags on cap": " - {name: r, target: {capability: [SYS_ADMIN], flags: [CLONE_NEWNS]}, action: deny}\n", + "flags on mount": " - {name: r, target: {syscall: [mount], flags: [CLONE_NEWNS]}, action: deny}\n", + "bad syscall": " - {name: r, target: {syscall: [read]}, action: deny}\n", + "cap prefix": " - {name: r, target: {capability: [CAP_SYS_ADMIN]}, action: deny}\n", + "bad action": " - {name: r, target: {syscall: [mount]}, action: block}\n", + "long fstype": " - {name: r, target: {mountFsType: [abcdefghijklmnop]}, action: deny}\n", + "duplicate name": " - {name: r, target: {syscall: [mount]}, action: deny}\n - {name: r, target: {syscall: [bpf]}, action: deny}\n", + "unknown field": " - {name: r, target: {path: [/x]}, action: deny}\n", + } { + if _, err := parse(base + rule); err == nil { + t.Errorf("%s: expected error", name) + } + } + if _, err := CompileKernel(mustParse(t, base+" - {name: r, target: {syscall: [mount]}, action: deny}\n"), 0); err == nil { + t.Error("zero generation must fail") + } +} + +func TestCompileSlotsMergesPolicies(t *testing.T) { + a := mustParse(t, "version: 1\nname: a\npolicy:\n mode: audit\n privilege:\n default: deny\n rules:\n - {name: no-bpf, target: {syscall: [bpf]}, action: deny, mode: enforce}\n") + b := mustParse(t, "version: 1\nname: b\npolicy:\n mode: enforce\n privilege:\n default: allow\n rules:\n - {name: mount-ok, target: {syscall: [mount]}, action: allow}\n - {name: bpf-ok, target: {syscall: [bpf]}, action: allow}\n") + c, err := CompileSlots([][]Document{{a, b}, {b}}, 4) + if err != nil { + t.Fatal(err) + } + if c.Slots != 2 { + t.Fatalf("slots %d", c.Slots) + } + get := func(k KernelKey) KernelValue { + v, ok := lookup(c, k) + if !ok { + t.Fatalf("missing %+v", k) + } + return v + } + // Merged default: deny (a denies); mode audit, because only a agrees. + if v := get(KernelKey{Slot: 0, Kind: KindDefault, ID: KindSyscall}); v.Rank != RankDenyAudit { + t.Fatalf("merged default rank %d", v.Rank) + } + // b's allow beats a's default deny; a's explicit deny beats b's allow. + if v := get(KernelKey{Slot: 0, Kind: KindSyscall, ID: Syscalls["mount"]}); v.Rank != RankAllow { + t.Fatalf("mount rank %d", v.Rank) + } + if v := get(KernelKey{Slot: 0, Kind: KindSyscall, ID: Syscalls["bpf"]}); v.Rank != RankDenyEnforce || c.RuleNames[v.RuleID] != "a/no-bpf" { + t.Fatalf("bpf %+v %s", v, c.RuleNames[v.RuleID]) + } + // Slot 1 holds b alone. + if v := get(KernelKey{Slot: 1, Kind: KindDefault, ID: KindSyscall}); v.Rank != RankAllow { + t.Fatalf("slot 1 default rank %d", v.Rank) + } + if v := get(KernelKey{Slot: 1, Kind: KindMeta}); v.Generation != 4 || v.Mode != KernelModeEnforce { + t.Fatalf("slot 1 meta %+v", v) + } +} + +// Policy A: audit deny mount. Policy B: enforce default deny, no mount rule. +// The audit rule must not weaken B's enforce default deny: mount stays blocked. +func TestAuditDenyNeverWeakensEnforceDefaultDeny(t *testing.T) { + a := mustParse(t, "version: 1\nname: a\npolicy:\n mode: audit\n privilege:\n default: allow\n rules:\n - {name: watch-mount, target: {syscall: [mount]}, action: deny}\n") + b := mustParse(t, "version: 1\nname: b\npolicy:\n mode: enforce\n privilege:\n default: deny\n") + c, err := CompileSlots([][]Document{{a, b}}, 1) + if err != nil { + t.Fatal(err) + } + v, ok := lookup(c, KernelKey{Kind: KindSyscall, ID: Syscalls["mount"]}) + if !ok || v.Rank != RankDenyEnforce { + t.Fatalf("mount must stay blocked: %+v", v) + } + // Without policy A the result is the same: adding an audit policy never + // makes the merged result less strict. + alone, _ := CompileSlots([][]Document{{b}}, 1) + if d, _ := lookup(alone, KernelKey{Kind: KindDefault, ID: KindSyscall}); d.Rank != RankDenyEnforce { + t.Fatalf("b alone: %+v", d) + } + // An explicit allow still beats another policy's default deny. + allow := mustParse(t, "version: 1\nname: c\npolicy:\n mode: enforce\n privilege:\n default: allow\n rules:\n - {name: mount-ok, target: {syscall: [mount]}, action: allow}\n") + c, _ = CompileSlots([][]Document{{allow, b}}, 1) + if v, _ := lookup(c, KernelKey{Kind: KindSyscall, ID: Syscalls["mount"]}); v.Rank != RankAllow { + t.Fatalf("allow must beat a default deny: %+v", v) + } + // One policy alone keeps its own rule mode: its audit rule decides. + single := mustParse(t, "version: 1\npolicy:\n mode: enforce\n privilege:\n default: deny\n rules:\n - {name: watch-mount, target: {syscall: [mount]}, action: deny, mode: audit}\n") + c, _ = CompileSlots([][]Document{{single}}, 1) + if v, _ := lookup(c, KernelKey{Kind: KindSyscall, ID: Syscalls["mount"]}); v.Rank != RankDenyAudit { + t.Fatalf("single policy rule mode: %+v", v) + } +} + +// Namespace syscalls combine {syscall, 0} with each flag key in the program. +// The audit-deny rule must not turn calls that both policies allow or only +// audit into an enforced deny. +func TestAuditDenyRaiseRespectsFlagKeys(t *testing.T) { + rankOf := func(c Compiled, flag string) uint8 { + anyFlag, _ := lookup(c, KernelKey{Kind: KindSyscall, ID: Syscalls["unshare"]}) + f, _ := lookup(c, KernelKey{Kind: KindSyscall, ID: Syscalls["unshare"], Qual: NamespaceFlags[flag]}) + return max(anyFlag.Rank, f.Rank) + } + // A: audit deny unshare NEWNS. B: enforce default deny, allow unshare (any flag). + a := mustParse(t, "version: 1\nname: a\npolicy:\n mode: audit\n privilege:\n default: allow\n rules:\n - {name: watch-mntns, target: {syscall: [unshare], flags: [CLONE_NEWNS]}, action: deny}\n") + b := mustParse(t, "version: 1\nname: b\npolicy:\n mode: enforce\n privilege:\n default: deny\n rules:\n - {name: unshare-ok, target: {syscall: [unshare]}, action: allow}\n") + c, err := CompileSlots([][]Document{{a, b}}, 1) + if err != nil { + t.Fatal(err) + } + if r := rankOf(c, "CLONE_NEWNS"); r != RankDenyAudit { + t.Fatalf("unshare -m: rank %d, want audit (B allows, A audits)", r) + } + // Mirror: A audit deny unshare (any flag). B: enforce default deny, allow NEWNS. + a = mustParse(t, "version: 1\nname: a\npolicy:\n mode: audit\n privilege:\n default: allow\n rules:\n - {name: watch-unshare, target: {syscall: [unshare]}, action: deny}\n") + b = mustParse(t, "version: 1\nname: b\npolicy:\n mode: enforce\n privilege:\n default: deny\n rules:\n - {name: mntns-ok, target: {syscall: [unshare], flags: [CLONE_NEWNS]}, action: allow}\n") + c, err = CompileSlots([][]Document{{a, b}}, 1) + if err != nil { + t.Fatal(err) + } + if r := rankOf(c, "CLONE_NEWNS"); r == RankDenyEnforce { + t.Fatal("unshare -m is allowed by B and audited by A: it must not be enforced") + } + // B's default deny still applies to flags B does not allow. + if r := rankOf(c, "CLONE_NEWNET"); r != RankDenyEnforce { + t.Fatalf("unshare --net: rank %d, want enforce from B's default", r) + } +} diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go b/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go new file mode 100644 index 0000000000..f105c652ae --- /dev/null +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go @@ -0,0 +1,132 @@ +// Copyright 2026 The Kubescape Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package privpolicy parses and compiles the privilege section of a sandbox +// policy into the enforce_privilege map entries. +package privpolicy + +import ( + "fmt" + "strings" +) + +const MaxRules = 128 + +type Document struct { + // Name qualifies rule names in events when several policies merge. + Name string `yaml:"name,omitempty" json:"name,omitempty"` + Version int `yaml:"version" json:"version"` + Policy Policy `yaml:"policy" json:"policy"` +} + +type Policy struct { + Mode string `yaml:"mode" json:"mode"` + Privilege Section `yaml:"privilege" json:"privilege"` +} + +type Section struct { + Default string `yaml:"default" json:"default"` + Rules []Rule `yaml:"rules,omitempty" json:"rules,omitempty"` +} + +type Rule struct { + Name string `yaml:"name" json:"name"` + Target Target `yaml:"target" json:"target"` + Action string `yaml:"action" json:"action"` + Mode string `yaml:"mode,omitempty" json:"mode,omitempty"` + Origin string `yaml:"origin,omitempty" json:"origin,omitempty"` +} + +// Target selects exactly one primitive kind. Flags qualify Syscall only. +type Target struct { + Syscall []string `yaml:"syscall,omitempty" json:"syscall,omitempty"` + Flags []string `yaml:"flags,omitempty" json:"flags,omitempty"` + Capability []string `yaml:"capability,omitempty" json:"capability,omitempty"` + MountFSType []string `yaml:"mountFsType,omitempty" json:"mountFsType,omitempty"` +} + +func validMode(m string, allowEmpty bool) bool { + return m == "audit" || m == "enforce" || (allowEmpty && m == "") +} + +func validAction(a string) bool { return a == "allow" || a == "deny" } + +func (d Document) Validate() error { + if d.Version != 1 { + return fmt.Errorf("unsupported policy version %d", d.Version) + } + p := d.Policy + if p.Mode != "off" && !validMode(p.Mode, false) { + return fmt.Errorf("policy mode must be audit, enforce or off") + } + s := p.Privilege + if !validAction(s.Default) { + return fmt.Errorf("privilege default must be allow or deny") + } + if p.Mode == "off" && len(s.Rules) > 0 { + return fmt.Errorf("an off policy cannot carry rules") + } + if len(s.Rules) > MaxRules { + return fmt.Errorf("too many privilege rules (max %d)", MaxRules) + } + names := map[string]bool{} + for _, r := range s.Rules { + if strings.TrimSpace(r.Name) == "" || names[r.Name] { + return fmt.Errorf("rule names must be unique and non-empty") + } + names[r.Name] = true + if !validAction(r.Action) { + return fmt.Errorf("rule %q: action must be allow or deny", r.Name) + } + if !validMode(r.Mode, true) { + return fmt.Errorf("rule %q: mode must be audit or enforce", r.Name) + } + if err := r.Target.validate(); err != nil { + return fmt.Errorf("rule %q: %w", r.Name, err) + } + } + return nil +} + +func (t Target) validate() error { + kinds := 0 + for _, n := range []int{len(t.Syscall), len(t.Capability), len(t.MountFSType)} { + if n > 0 { + kinds++ + } + } + if kinds != 1 { + return fmt.Errorf("target must select exactly one of syscall, capability or mountFsType") + } + for _, s := range t.Syscall { + if _, ok := Syscalls[s]; !ok { + return fmt.Errorf("unsupported syscall %q", s) + } + } + if len(t.Flags) > 0 { + if len(t.Syscall) == 0 { + return fmt.Errorf("flags qualify a syscall target only") + } + for _, s := range t.Syscall { + if !NamespaceSyscalls[s] { + return fmt.Errorf("flags qualify unshare, setns, clone and clone3 only") + } + } + for _, f := range t.Flags { + if _, ok := NamespaceFlags[f]; !ok { + return fmt.Errorf("unsupported flag %q", f) + } + } + } + for _, c := range t.Capability { + if _, ok := Capabilities[c]; !ok { + return fmt.Errorf("unsupported capability %q (use the short name, for example SYS_ADMIN)", c) + } + } + for _, fs := range t.MountFSType { + if fs == "" || len(fs) >= KernelNameLen { + return fmt.Errorf("filesystem type %q must be 1-%d bytes", fs, KernelNameLen-1) + } + } + return nil +} diff --git a/pkg/ebpf/gadgets/enforce_privilege/privilege_policy.h b/pkg/ebpf/gadgets/enforce_privilege/privilege_policy.h new file mode 100644 index 0000000000..ba0b8bf2d7 --- /dev/null +++ b/pkg/ebpf/gadgets/enforce_privilege/privilege_policy.h @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright 2026 The Kubescape Authors + +// Policy map of the privilege section. All access to the map goes through this +// header (see ../common/policy.h for the resolution rules). + +#ifndef __PRIVILEGE_POLICY_H +#define __PRIVILEGE_POLICY_H + +#include +#include + +// A task that moves into a child cgroup of its container stays covered, up to +// this many levels below the bound cgroup. +#define BINDINGS_CGROUP_DEPTH 32 + +#include "../common/bindings.h" +#include "../common/policy.h" + +#define PRIV_POLICY_MAX_ENTRIES 16384 +#define PRIV_NAME_LEN 16 + +enum priv_kind : __u32 { + // id 0: generation and policy mode (value.generation, value.mode). + PRIV_KIND_META = 0, + // id = target kind: default rank, rule id and origin id for that kind. + PRIV_KIND_DEFAULT = 1, + PRIV_KIND_SYSCALL = 2, // id: priv_syscall, qual: CLONE_NEW* flag or 0 (any) + PRIV_KIND_CAPABILITY = 3, // id: capability number + PRIV_KIND_MOUNT_FSTYPE = 4, // name: filesystem type +}; + +struct priv_key { + __u32 slot; + __u32 kind; + __u32 id; + __u32 pad; + __u64 qual; + char name[PRIV_NAME_LEN]; +}; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, PRIV_POLICY_MAX_ENTRIES); + __uint(map_flags, BPF_F_RDONLY_PROG | BPF_F_NO_PREALLOC); + __type(key, struct priv_key); + __type(value, struct policy_value); +} policy SEC(".maps"); + +// priv_header returns the META entry of the slot for a task in a bound cgroup, +// or NULL when nothing must be judged. +static __always_inline struct policy_value *priv_header(__u64 *cgroup, + __u32 *slot) +{ + struct priv_key key = { .kind = PRIV_KIND_META }; + struct policy_value *header; + + if (!binding_lookup(cgroup, slot)) + return NULL; + key.slot = *slot; + header = bpf_map_lookup_elem(&policy, &key); + if (!header || header->mode == POLICY_MODE_OFF) + return NULL; + return header; +} + +static __always_inline void priv_consider(const struct priv_key *key, + struct policy_match *m) +{ + policy_take(m, bpf_map_lookup_elem(&policy, key), false); +} + +// priv_finish_kind applies the default of kind when no rule of that kind +// matched. Each kind resolves on its own; merge kinds with policy_merge. +static __always_inline void priv_finish_kind(__u32 slot, __u32 kind, + struct policy_match *m) +{ + struct priv_key key = { .slot = slot, .kind = PRIV_KIND_DEFAULT, .id = kind }; + + if (m->rank == POLICY_RANK_NONE) + policy_take(m, bpf_map_lookup_elem(&policy, &key), true); +} + +#endif /* __PRIVILEGE_POLICY_H */ diff --git a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c new file mode 100644 index 0000000000..e731de0ab0 --- /dev/null +++ b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c @@ -0,0 +1,823 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright 2026 The Kubescape Authors + +// enforce_privilege: enforce the privilege section of a SandboxPolicy with BPF +// LSM. The program is fixed; the policy is map data (privilege_policy.h), +// frozen per instance; a rule change starts a replacement instance. Audit and +// enforce share one code path: the rank of the deciding entry carries the mode. +// +// Notes that apply to every hook: +// - Only armed tasks are judged (see arming.h). +// - A deny returns -EPERM from the LSM hook, before the side effect. +// - The kernel calls the next BPF LSM program only while every earlier one +// returned 0, so the hooks do not read the previous return value. They also +// name only the hook arguments they need, because the trailing arguments of +// some hooks changed between kernel versions. +// - Each hook body is a plain function, so it can also be attached as +// fmod_ret on the matching security_* function. + +#include +#include +#include +#include + +#include + +#include "../common/arming.h" +#include "../common/verdict.h" +#include "privilege_policy.h" + +char LICENSE[] SEC("license") = "GPL"; + +GADGET_PARAM(host_mntns_id); + +#define EPERM 1 +#define CAP_SYS_ADMIN 21 +#define CAP_OPT_NOAUDIT (1UL << 1) +#define PTRACE_MODE_ATTACH 0x02 + +#define CLONE_NEWTIME 0x00000080ULL +#define CLONE_NEWNS 0x00020000ULL +#define CLONE_NEWCGROUP 0x02000000ULL +#define CLONE_NEWUTS 0x04000000ULL +#define CLONE_NEWIPC 0x08000000ULL +#define CLONE_NEWUSER 0x10000000ULL +#define CLONE_NEWPID 0x20000000ULL +#define CLONE_NEWNET 0x40000000ULL +#define CLONE_NS_MASK \ + (CLONE_NEWTIME | CLONE_NEWNS | CLONE_NEWCGROUP | CLONE_NEWUTS | \ + CLONE_NEWIPC | CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNET) + +// Architecture-independent syscall ids. The values are part of the map ABI. +enum priv_syscall : __u32 { + PRIV_SC_NONE = 0, + PRIV_SC_UNSHARE = 1, + PRIV_SC_SETNS = 2, + PRIV_SC_CLONE = 3, + PRIV_SC_CLONE3 = 4, + PRIV_SC_MOUNT = 5, + PRIV_SC_UMOUNT2 = 6, + PRIV_SC_PIVOT_ROOT = 7, + PRIV_SC_CHROOT = 8, + PRIV_SC_OPEN_TREE = 9, + PRIV_SC_MOVE_MOUNT = 10, + PRIV_SC_FSOPEN = 11, + PRIV_SC_FSMOUNT = 12, + PRIV_SC_FSPICK = 13, + PRIV_SC_MOUNT_SETATTR = 14, + PRIV_SC_PTRACE = 15, + PRIV_SC_PROCESS_VM_READV = 16, + PRIV_SC_PROCESS_VM_WRITEV = 17, + PRIV_SC_INIT_MODULE = 18, + PRIV_SC_FINIT_MODULE = 19, + PRIV_SC_BPF = 20, + PRIV_SC_PERF_EVENT_OPEN = 21, + PRIV_SC_KEXEC_LOAD = 22, + PRIV_SC_KEYCTL = 23, + PRIV_SC_USERFAULTFD = 24, + PRIV_SC_IO_URING_SETUP = 25, + PRIV_SC_MKNOD = 26, + PRIV_SC_SETUID = 27, + PRIV_SC_SETGID = 28, + PRIV_SC_CAPSET = 29, +}; + +enum priv_hook : __u32 { + PRIV_HOOK_USERNS_CREATE, + PRIV_HOOK_TASK_ALLOC, + PRIV_HOOK_CAPABLE, + PRIV_HOOK_SB_MOUNT, + PRIV_HOOK_SB_KERN_MOUNT, + PRIV_HOOK_SB_UMOUNT, + PRIV_HOOK_MOVE_MOUNT, + PRIV_HOOK_SB_PIVOTROOT, + PRIV_HOOK_PATH_CHROOT, + PRIV_HOOK_ANON_INODE, + PRIV_HOOK_PTRACE_ACCESS_CHECK, + PRIV_HOOK_PTRACE_TRACEME, + PRIV_HOOK_KERNEL_READ_FILE, + PRIV_HOOK_KERNEL_LOAD_DATA, + PRIV_HOOK_KERNEL_MODULE_REQUEST, + PRIV_HOOK_BPF, + PRIV_HOOK_PERF_EVENT_OPEN, + PRIV_HOOK_KEY_ALLOC, + PRIV_HOOK_KEY_PERMISSION, + PRIV_HOOK_INODE_MKNOD, + PRIV_HOOK_TASK_FIX_SETUID, + PRIV_HOOK_TASK_FIX_SETGID, + PRIV_HOOK_CAPSET, +}; + +struct event { + VERDICT_BASE_FIELDS; + __u32 host_pid; + // Rule and origin ids of the deciding rule or default. + __u32 rule_id; + __u32 origin_id; + // 1 when a section default decided, not a rule. + __u32 is_default; + char comm[16]; + enum priv_hook hook_raw; + enum priv_syscall syscall_raw; + // CLONE_NEW* flags of a namespace call. + __u64 flags; + // Capability number; -1 when the hook is not a capability check. + __s32 capability; + char fstype[PRIV_NAME_LEN]; + // Policy slot (combination of policies) of the bound container. + __u32 slot; +}; + +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 4096); +} events SEC(".maps"); +GADGET_TRACER(verdict, events, event); + +// What a hook tried to do. Used for the event only. +struct priv_target { + enum priv_syscall sc; + __u64 flags; + __s32 cap; + char fstype[PRIV_NAME_LEN]; +}; + +/* Syscall numbers */ + +static __always_inline long current_syscall_nr(void) +{ + struct pt_regs *regs = + (struct pt_regs *)bpf_task_pt_regs(bpf_get_current_task_btf()); + +#if defined(__TARGET_ARCH_arm64) + return BPF_CORE_READ(regs, syscallno); +#elif defined(__TARGET_ARCH_x86) + return BPF_CORE_READ(regs, orig_ax); +#else +#error "unsupported architecture" +#endif +} + +// A 32-bit (compat) task uses other syscall numbers. Hooks that decide by +// syscall number fail closed for it: they apply every candidate syscall. +static __always_inline bool current_is_compat(void) +{ + struct task_struct *task = bpf_get_current_task_btf(); + +#if defined(__TARGET_ARCH_arm64) + return BPF_CORE_READ(task, thread_info.flags) & (1UL << 22); // TIF_32BIT +#else + // TS_COMPAT (ia32), or an x32 syscall (__X32_SYSCALL_BIT in the number). + return (BPF_CORE_READ(task, thread_info.status) & 0x0002) || + (current_syscall_nr() & 0x40000000); +#endif +} + +static __always_inline __u64 current_syscall_arg(int n) +{ + struct pt_regs *regs = + (struct pt_regs *)bpf_task_pt_regs(bpf_get_current_task_btf()); + + if (n == 0) + return PT_REGS_PARM1_CORE_SYSCALL(regs); + return PT_REGS_PARM2_CORE_SYSCALL(regs); +} + +// Native syscall number -> priv_syscall. Compat (32-bit) numbers are not +// mapped. +static __always_inline enum priv_syscall priv_syscall_from_nr(long nr) +{ + if (current_is_compat()) + return PRIV_SC_NONE; + switch (nr) { + // Same number on every architecture. + case 425: + return PRIV_SC_IO_URING_SETUP; + case 428: + return PRIV_SC_OPEN_TREE; + case 429: + return PRIV_SC_MOVE_MOUNT; + case 430: + return PRIV_SC_FSOPEN; + case 432: + return PRIV_SC_FSMOUNT; + case 433: + return PRIV_SC_FSPICK; + case 435: + return PRIV_SC_CLONE3; + case 442: + return PRIV_SC_MOUNT_SETATTR; + case 467: // open_tree_attr + return PRIV_SC_OPEN_TREE; +#if defined(__TARGET_ARCH_arm64) + case 97: + return PRIV_SC_UNSHARE; + case 268: + return PRIV_SC_SETNS; + case 220: + return PRIV_SC_CLONE; + case 40: + return PRIV_SC_MOUNT; + case 39: + return PRIV_SC_UMOUNT2; + case 41: + return PRIV_SC_PIVOT_ROOT; + case 51: + return PRIV_SC_CHROOT; + case 117: + return PRIV_SC_PTRACE; + case 270: + return PRIV_SC_PROCESS_VM_READV; + case 271: + return PRIV_SC_PROCESS_VM_WRITEV; + case 105: + return PRIV_SC_INIT_MODULE; + case 273: + return PRIV_SC_FINIT_MODULE; + case 104: + case 294: + return PRIV_SC_KEXEC_LOAD; + case 33: // mknodat; arm64 has no mknod + return PRIV_SC_MKNOD; +#elif defined(__TARGET_ARCH_x86) + case 272: + return PRIV_SC_UNSHARE; + case 308: + return PRIV_SC_SETNS; + case 56: + return PRIV_SC_CLONE; + case 165: + return PRIV_SC_MOUNT; + case 166: + return PRIV_SC_UMOUNT2; + case 155: + return PRIV_SC_PIVOT_ROOT; + case 161: + return PRIV_SC_CHROOT; + case 101: + return PRIV_SC_PTRACE; + case 310: + return PRIV_SC_PROCESS_VM_READV; + case 311: + return PRIV_SC_PROCESS_VM_WRITEV; + case 175: + return PRIV_SC_INIT_MODULE; + case 313: + return PRIV_SC_FINIT_MODULE; + case 246: + case 320: + return PRIV_SC_KEXEC_LOAD; + case 133: + case 259: + return PRIV_SC_MKNOD; +#endif + } + return PRIV_SC_NONE; +} + +/* Resolution */ + +static __always_inline void consider_syscall(__u32 slot, enum priv_syscall sc, + __u64 qual, + struct policy_match *m) +{ + struct priv_key key = { .slot = slot }; + + key.kind = PRIV_KIND_SYSCALL; + key.id = sc; + key.qual = qual; + priv_consider(&key, m); +} + +// resolve_syscall resolves the syscall kind: rules on the syscall with any +// flag, rules on each CLONE_NEW* flag in flags, then the syscall default. +static __always_inline void resolve_syscall(__u32 slot, enum priv_syscall sc, + __u64 flags, struct policy_match *out) +{ + static const __u64 ns_flags[] = { + CLONE_NEWNS, CLONE_NEWCGROUP, CLONE_NEWUTS, CLONE_NEWIPC, + CLONE_NEWUSER, CLONE_NEWPID, CLONE_NEWNET, CLONE_NEWTIME, + }; + struct policy_match m = {}; + + consider_syscall(slot, sc, 0, &m); +#pragma unroll + for (int i = 0; i < sizeof(ns_flags) / sizeof(ns_flags[0]); i++) { + if (flags & ns_flags[i]) + consider_syscall(slot, sc, ns_flags[i], &m); + } + priv_finish_kind(slot, PRIV_KIND_SYSCALL, &m); + policy_merge(out, &m); +} + +static __always_inline void resolve_capability(__u32 slot, int cap, + struct policy_match *out) +{ + struct priv_key key = { .slot = slot }; + struct policy_match m = {}; + + key.kind = PRIV_KIND_CAPABILITY; + key.id = cap; + priv_consider(&key, &m); + priv_finish_kind(slot, PRIV_KIND_CAPABILITY, &m); + policy_merge(out, &m); +} + +static __always_inline void resolve_fstype(__u32 slot, const char *fstype, + struct policy_match *out) +{ + struct priv_key key = { .slot = slot }; + struct policy_match m = {}; + + key.kind = PRIV_KIND_MOUNT_FSTYPE; + __builtin_memcpy(key.name, fstype, PRIV_NAME_LEN); + priv_consider(&key, &m); + priv_finish_kind(slot, PRIV_KIND_MOUNT_FSTYPE, &m); + policy_merge(out, &m); +} + +/* Scope and verdict */ + +// priv_scope returns the policy header for an armed task in a bound cgroup, +// or NULL. +static __always_inline struct policy_value *priv_scope(__u64 *cgid, __u32 *slot) +{ + if (!arming_task_armed(bpf_get_current_task_btf())) + return NULL; + return priv_header(cgid, slot); +} + +static __always_inline int priv_verdict(void *ctx, enum priv_hook hook, + const struct priv_target *t, + const struct policy_value *header, + __u64 cgid, __u32 slot, + const struct policy_match *m) +{ + struct event *e; + bool reporting; + + if (m->rank < POLICY_RANK_DENY_AUDIT) { + if (m->rank == POLICY_RANK_ALLOW) + verdict_count(VERDICT_COUNT_ALLOWED); + return 0; + } + verdict_count(VERDICT_COUNT_DENIED); + + reporting = verdict_report_allowed(); + e = reporting ? bpf_ringbuf_reserve(&events, sizeof(*e), 0) : NULL; + if (e) { + VERDICT_FILL_BASE(e, m, header->generation, cgid); + e->host_pid = bpf_get_current_pid_tgid() >> 32; + e->rule_id = m->rule_id; + e->origin_id = m->origin_id; + e->is_default = m->is_default; + e->slot = slot; + bpf_get_current_comm(e->comm, sizeof(e->comm)); + e->hook_raw = hook; + e->syscall_raw = t->sc; + e->flags = t->flags; + e->capability = t->cap; + __builtin_memcpy(e->fstype, t->fstype, PRIV_NAME_LEN); + bpf_ringbuf_submit(e, 0); + verdict_count(VERDICT_COUNT_SUBMITTED); + } else { + verdict_count(reporting ? VERDICT_COUNT_DROPPED : + VERDICT_COUNT_RATE_LIMITED); + } + + // Keep explicit signed errno branches visible to the verifier. + volatile int result = 0; + if (m->rank == POLICY_RANK_DENY_ENFORCE) + result = -EPERM; + return result; +} + +// check_syscall is the common body of hooks that map to one syscall. +static __always_inline int check_syscall(void *ctx, enum priv_hook hook, + enum priv_syscall sc, __u64 flags) +{ + struct priv_target t = { .sc = sc, .flags = flags, .cap = -1 }; + struct policy_match m = {}; + __u64 cgid = 0; + __u32 slot = 0; + struct policy_value *header; + + if (sc == PRIV_SC_NONE) + return 0; + header = priv_scope(&cgid, &slot); + if (!header) + return 0; + resolve_syscall(slot, sc, flags, &m); + return priv_verdict(ctx, hook, &t, header, cgid, slot, &m); +} + +/* Arming */ + +SEC("lsm/bprm_committed_creds") +int BPF_PROG(ig_priv_exec) +{ + arming_on_exec(); + return 0; +} + +/* Namespaces */ + +static __always_inline int do_userns_create(void *ctx) +{ + enum priv_syscall sc = priv_syscall_from_nr(current_syscall_nr()); + struct priv_target t = { .flags = CLONE_NEWUSER, .cap = -1 }; + struct policy_match m = {}; + __u64 cgid = 0; + __u32 slot = 0; + struct policy_value *header; + + header = priv_scope(&cgid, &slot); + if (!header) + return 0; + + if (sc == PRIV_SC_UNSHARE || sc == PRIV_SC_CLONE || + sc == PRIV_SC_CLONE3) { + t.sc = sc; + resolve_syscall(slot, sc, CLONE_NEWUSER, &m); + } else { + // Unknown entry point (for example a compat syscall): apply the + // most limiting rule of the three namespace-creating syscalls. + t.sc = PRIV_SC_UNSHARE; + resolve_syscall(slot, PRIV_SC_UNSHARE, CLONE_NEWUSER, &m); + resolve_syscall(slot, PRIV_SC_CLONE, CLONE_NEWUSER, &m); + resolve_syscall(slot, PRIV_SC_CLONE3, CLONE_NEWUSER, &m); + } + return priv_verdict(ctx, PRIV_HOOK_USERNS_CREATE, &t, header, cgid, slot, &m); +} + +SEC("lsm/userns_create") +int BPF_PROG(ig_priv_userns, const struct cred *cred) +{ + return do_userns_create(ctx); +} + +static __always_inline int do_task_alloc(void *ctx, struct task_struct *task, + unsigned long clone_flags) +{ + enum priv_syscall sc; + __u64 flags; + + arming_on_fork(task); + + // userns_create judges CLONE_NEWUSER. + flags = clone_flags & CLONE_NS_MASK & ~CLONE_NEWUSER; + if (current_is_compat() && flags) { + // Unknown syscall: apply clone and clone3 rules, all flags. + struct priv_target t = { .sc = PRIV_SC_CLONE3, .flags = flags, .cap = -1 }; + struct policy_match m = {}; + struct policy_value *header; + __u64 cgid = 0; + __u32 slot = 0; + + header = priv_scope(&cgid, &slot); + if (!header) + return 0; + resolve_syscall(slot, PRIV_SC_CLONE, flags, &m); + resolve_syscall(slot, PRIV_SC_CLONE3, flags, &m); + return priv_verdict(ctx, PRIV_HOOK_TASK_ALLOC, &t, header, cgid, slot, &m); + } + sc = priv_syscall_from_nr(current_syscall_nr()) == PRIV_SC_CLONE3 ? + PRIV_SC_CLONE3 : + PRIV_SC_CLONE; + // In clone(2) the CLONE_NEWTIME bit is part of the exit signal. + if (sc == PRIV_SC_CLONE) + flags &= ~CLONE_NEWTIME; + if (!flags) + return 0; + return check_syscall(ctx, PRIV_HOOK_TASK_ALLOC, sc, flags); +} + +SEC("lsm/task_alloc") +int BPF_PROG(ig_priv_task_alloc, struct task_struct *task, + unsigned long clone_flags) +{ + return do_task_alloc(ctx, task, clone_flags); +} + +/* Capabilities */ + +// Syscalls whose only LSM hook is the CAP_SYS_ADMIN check, and the CLONE_NEW* +// flags they act on. +static __always_inline enum priv_syscall capable_syscall(__u64 *flags) +{ + enum priv_syscall sc = priv_syscall_from_nr(current_syscall_nr()); + + *flags = 0; + switch (sc) { + case PRIV_SC_UNSHARE: + *flags = current_syscall_arg(0) & CLONE_NS_MASK & ~CLONE_NEWUSER; + return sc; + case PRIV_SC_SETNS: + *flags = current_syscall_arg(1) & CLONE_NS_MASK; + // nstype 0 accepts any namespace type: apply every flag rule. + if (!*flags) + *flags = CLONE_NS_MASK; + return sc; + case PRIV_SC_OPEN_TREE: + case PRIV_SC_FSOPEN: + case PRIV_SC_FSMOUNT: + case PRIV_SC_FSPICK: + case PRIV_SC_MOUNT_SETATTR: + return sc; + default: + // mount, umount2, pivot_root, chroot and move_mount have their + // own hooks. + return PRIV_SC_NONE; + } +} + +static __always_inline int do_capable(void *ctx, const struct cred *cred, + int cap, unsigned int opts) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct priv_target t = { .cap = cap }; + struct policy_match m = {}; + __u64 cgid = 0; + __u32 slot = 0; + struct policy_value *header; + + // Kernel probes that do not grant an action (for example the overcommit + // check). + if (opts & CAP_OPT_NOAUDIT) + return 0; + // Overridden credentials, for example overlayfs copy-up with the + // mounter's credentials. Denying those breaks writes to the container + // root filesystem. + if ((void *)cred != (void *)task->real_cred) + return 0; + header = priv_scope(&cgid, &slot); + if (!header) + return 0; + + resolve_capability(slot, cap, &m); + if (cap == CAP_SYS_ADMIN && current_is_compat()) { + // Unknown syscall: apply every syscall judged at this check. + t.flags = CLONE_NS_MASK & ~CLONE_NEWUSER; + resolve_syscall(slot, PRIV_SC_UNSHARE, t.flags, &m); + resolve_syscall(slot, PRIV_SC_SETNS, CLONE_NS_MASK, &m); + resolve_syscall(slot, PRIV_SC_OPEN_TREE, 0, &m); + resolve_syscall(slot, PRIV_SC_FSOPEN, 0, &m); + resolve_syscall(slot, PRIV_SC_FSMOUNT, 0, &m); + resolve_syscall(slot, PRIV_SC_FSPICK, 0, &m); + resolve_syscall(slot, PRIV_SC_MOUNT_SETATTR, 0, &m); + } else if (cap == CAP_SYS_ADMIN) { + t.sc = capable_syscall(&t.flags); + if (t.sc != PRIV_SC_NONE) + resolve_syscall(slot, t.sc, t.flags, &m); + } + return priv_verdict(ctx, PRIV_HOOK_CAPABLE, &t, header, cgid, slot, &m); +} + +SEC("lsm/capable") +int BPF_PROG(ig_priv_capable, const struct cred *cred, + struct user_namespace *ns, int cap, unsigned int opts) +{ + return do_capable(ctx, cred, cap, opts); +} + +SEC("lsm/capset") +int BPF_PROG(ig_priv_capset) +{ + return check_syscall(ctx, PRIV_HOOK_CAPSET, PRIV_SC_CAPSET, 0); +} + +SEC("lsm/task_fix_setuid") +int BPF_PROG(ig_priv_setuid) +{ + return check_syscall(ctx, PRIV_HOOK_TASK_FIX_SETUID, PRIV_SC_SETUID, 0); +} + +SEC("lsm/task_fix_setgid") +int BPF_PROG(ig_priv_setgid) +{ + return check_syscall(ctx, PRIV_HOOK_TASK_FIX_SETGID, PRIV_SC_SETGID, 0); +} + +/* Mounts */ + +SEC("lsm/sb_mount") +int BPF_PROG(ig_priv_sb_mount) +{ + return check_syscall(ctx, PRIV_HOOK_SB_MOUNT, PRIV_SC_MOUNT, 0); +} + +// A new superblock, from mount(2) or from fsconfig(FSCONFIG_CMD_CREATE). +// Judging here also covers detached mounts (fsmount without move_mount). +static __always_inline int do_sb_kern_mount(void *ctx, + const struct super_block *sb) +{ + struct priv_target t = { .cap = -1 }; + struct policy_match m = {}; + __u64 cgid = 0; + __u32 slot = 0; + struct policy_value *header; + + header = priv_scope(&cgid, &slot); + if (!header) + return 0; + bpf_probe_read_kernel_str(t.fstype, sizeof(t.fstype), + BPF_CORE_READ(sb, s_type, name)); + t.sc = priv_syscall_from_nr(current_syscall_nr()); + resolve_fstype(slot, t.fstype, &m); + return priv_verdict(ctx, PRIV_HOOK_SB_KERN_MOUNT, &t, header, cgid, slot, &m); +} + +SEC("lsm/sb_kern_mount") +int BPF_PROG(ig_priv_sb_kern_mount, const struct super_block *sb) +{ + return do_sb_kern_mount(ctx, sb); +} + +SEC("lsm/sb_umount") +int BPF_PROG(ig_priv_sb_umount) +{ + return check_syscall(ctx, PRIV_HOOK_SB_UMOUNT, PRIV_SC_UMOUNT2, 0); +} + +SEC("lsm/move_mount") +int BPF_PROG(ig_priv_move_mount) +{ + return check_syscall(ctx, PRIV_HOOK_MOVE_MOUNT, PRIV_SC_MOVE_MOUNT, 0); +} + +SEC("lsm/sb_pivotroot") +int BPF_PROG(ig_priv_pivotroot) +{ + return check_syscall(ctx, PRIV_HOOK_SB_PIVOTROOT, PRIV_SC_PIVOT_ROOT, + 0); +} + +SEC("lsm/path_chroot") +int BPF_PROG(ig_priv_chroot) +{ + return check_syscall(ctx, PRIV_HOOK_PATH_CHROOT, PRIV_SC_CHROOT, 0); +} + +/* io_uring and userfaultfd */ + +static __always_inline int do_anon_inode(void *ctx, const struct qstr *name) +{ + char buf[16] = {}; + + bpf_probe_read_kernel_str(buf, sizeof(buf), BPF_CORE_READ(name, name)); + // "[io_uring]" + if (buf[0] == '[' && buf[1] == 'i' && buf[2] == 'o' && buf[3] == '_' && + buf[4] == 'u' && buf[9] == ']') + return check_syscall(ctx, PRIV_HOOK_ANON_INODE, + PRIV_SC_IO_URING_SETUP, 0); + // "[userfaultfd]" + if (buf[0] == '[' && buf[1] == 'u' && buf[2] == 's' && buf[3] == 'e' && + buf[4] == 'r' && buf[5] == 'f' && buf[12] == ']') + return check_syscall(ctx, PRIV_HOOK_ANON_INODE, + PRIV_SC_USERFAULTFD, 0); + return 0; +} + +SEC("lsm/inode_init_security_anon") +int BPF_PROG(ig_priv_anon_inode, struct inode *inode, const struct qstr *name) +{ + return do_anon_inode(ctx, name); +} + +/* ptrace */ + +static __always_inline int do_ptrace_access_check(void *ctx, unsigned int mode) +{ + enum priv_syscall sc = priv_syscall_from_nr(current_syscall_nr()); + struct priv_target t = { .sc = PRIV_SC_PTRACE, .cap = -1 }; + struct policy_match m = {}; + struct policy_value *header; + __u64 cgid = 0; + __u32 slot = 0; + + if (current_is_compat()) { + // Unknown syscall: apply all three ptrace-family syscalls. + header = priv_scope(&cgid, &slot); + if (!header) + return 0; + resolve_syscall(slot, PRIV_SC_PTRACE, 0, &m); + resolve_syscall(slot, PRIV_SC_PROCESS_VM_READV, 0, &m); + resolve_syscall(slot, PRIV_SC_PROCESS_VM_WRITEV, 0, &m); + return priv_verdict(ctx, PRIV_HOOK_PTRACE_ACCESS_CHECK, &t, + header, cgid, slot, &m); + } + + switch (sc) { + case PRIV_SC_PTRACE: + case PRIV_SC_PROCESS_VM_READV: + case PRIV_SC_PROCESS_VM_WRITEV: + break; + default: + // /proc//mem and similar attach-mode access count as + // ptrace. Read-mode /proc access (ps, top) does not. + if (!(mode & PTRACE_MODE_ATTACH)) + return 0; + sc = PRIV_SC_PTRACE; + } + return check_syscall(ctx, PRIV_HOOK_PTRACE_ACCESS_CHECK, sc, 0); +} + +SEC("lsm/ptrace_access_check") +int BPF_PROG(ig_priv_ptrace, struct task_struct *child, unsigned int mode) +{ + return do_ptrace_access_check(ctx, mode); +} + +SEC("lsm/ptrace_traceme") +int BPF_PROG(ig_priv_traceme) +{ + return check_syscall(ctx, PRIV_HOOK_PTRACE_TRACEME, PRIV_SC_PTRACE, 0); +} + +/* Kernel modules and kexec */ + +static __always_inline int do_kernel_read_file(void *ctx, int id) +{ + enum priv_syscall sc = PRIV_SC_NONE; + + if (id == bpf_core_enum_value(enum kernel_read_file_id, READING_MODULE)) + sc = PRIV_SC_FINIT_MODULE; + else if (id == bpf_core_enum_value(enum kernel_read_file_id, + READING_KEXEC_IMAGE) || + id == bpf_core_enum_value(enum kernel_read_file_id, + READING_KEXEC_INITRAMFS)) + sc = PRIV_SC_KEXEC_LOAD; + return check_syscall(ctx, PRIV_HOOK_KERNEL_READ_FILE, sc, 0); +} + +SEC("lsm/kernel_read_file") +int BPF_PROG(ig_priv_kernel_read_file, struct file *file, int id) +{ + return do_kernel_read_file(ctx, id); +} + +static __always_inline int do_kernel_load_data(void *ctx, int id) +{ + enum priv_syscall sc = PRIV_SC_NONE; + + if (id == bpf_core_enum_value(enum kernel_load_data_id, LOADING_MODULE)) + sc = PRIV_SC_INIT_MODULE; + else if (id == bpf_core_enum_value(enum kernel_load_data_id, + LOADING_KEXEC_IMAGE)) + sc = PRIV_SC_KEXEC_LOAD; + return check_syscall(ctx, PRIV_HOOK_KERNEL_LOAD_DATA, sc, 0); +} + +SEC("lsm/kernel_load_data") +int BPF_PROG(ig_priv_kernel_load_data, int id) +{ + return do_kernel_load_data(ctx, id); +} + +// An implicit module load requested on behalf of the task (for example an +// unknown socket family). It is judged as init_module. +SEC("lsm/kernel_module_request") +int BPF_PROG(ig_priv_module_request) +{ + return check_syscall(ctx, PRIV_HOOK_KERNEL_MODULE_REQUEST, + PRIV_SC_INIT_MODULE, 0); +} + +/* bpf, perf, keys, mknod */ + +SEC("lsm/bpf") +int BPF_PROG(ig_priv_bpf) +{ + return check_syscall(ctx, PRIV_HOOK_BPF, PRIV_SC_BPF, 0); +} + +SEC("lsm/perf_event_open") +int BPF_PROG(ig_priv_perf_event_open) +{ + return check_syscall(ctx, PRIV_HOOK_PERF_EVENT_OPEN, + PRIV_SC_PERF_EVENT_OPEN, 0); +} + +SEC("lsm/key_alloc") +int BPF_PROG(ig_priv_key_alloc) +{ + return check_syscall(ctx, PRIV_HOOK_KEY_ALLOC, PRIV_SC_KEYCTL, 0); +} + +SEC("lsm/key_permission") +int BPF_PROG(ig_priv_key_permission) +{ + return check_syscall(ctx, PRIV_HOOK_KEY_PERMISSION, PRIV_SC_KEYCTL, 0); +} + +// vfs_mknod also runs for bind(2) of a unix socket path and for overlayfs +// whiteouts (unlink or rename of a lower file). Judge only mknod(2) and +// mknodat(2). +SEC("lsm/inode_mknod") +int BPF_PROG(ig_priv_mknod) +{ + if (!current_is_compat() && + priv_syscall_from_nr(current_syscall_nr()) != PRIV_SC_MKNOD) + return 0; + return check_syscall(ctx, PRIV_HOOK_INODE_MKNOD, PRIV_SC_MKNOD, 0); +} diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go new file mode 100644 index 0000000000..8561476cf3 --- /dev/null +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go @@ -0,0 +1,132 @@ +// Copyright 2026 The Kubescape Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package runtime is the native owner of the enforce_privilege gadget. The +// binding controller comes from the common enforcement package; this package +// adds the privilege policy map and the operator registration. +package runtime + +import ( + "fmt" + + "github.com/cilium/ebpf" + "github.com/spf13/viper" + "golang.org/x/sys/unix" + + gadgetcontext "github.com/inspektor-gadget/inspektor-gadget/pkg/gadget-context" + "github.com/inspektor-gadget/inspektor-gadget/pkg/gadget-service/api" + "github.com/inspektor-gadget/inspektor-gadget/pkg/operators" + "github.com/inspektor-gadget/inspektor-gadget/pkg/params" + "github.com/kubescape/node-agent/pkg/ebpf/gadgets/common/enforcement" + privpolicy "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/policy" +) + +const ( + operatorName = "enforce-privilege" + annotation = "annotations.enforce-privilege" + configKey = "enforce-privilege/config" + bindingsKey = "enforce-privilege/bindings" +) + +type ContainerIdentity = enforcement.ContainerIdentity + +// Config holds the compiled rules and the initial container selection. +type Config struct { + Policy privpolicy.Compiled + InitialBindings []enforcement.Binding +} + +func WithConfig(config Config) gadgetcontext.Option { + config.Policy.Entries = append([]privpolicy.KernelEntry(nil), config.Policy.Entries...) + config.InitialBindings = append([]enforcement.Binding(nil), config.InitialBindings...) + return func(gc *gadgetcontext.GadgetContext) { gc.SetVar(configKey, config) } +} + +// GetBindings returns the controller of a running instance. +func GetBindings(gc operators.GadgetContext) (*enforcement.Bindings, error) { + value, _ := gc.GetVar(bindingsKey) + b, ok := value.(*enforcement.Bindings) + if !ok { + return nil, fmt.Errorf("privilege binding controller is unavailable") + } + return b, nil +} + +func validateConfig(config Config) error { + p := config.Policy + if p.Generation == 0 || p.Slots > privpolicy.KernelMaxSlots || len(p.Entries) > privpolicy.KernelMaxEntries { + return fmt.Errorf("invalid compiled privilege policy") + } + metas := uint32(0) + for _, e := range p.Entries { + if e.Key.Slot >= p.Slots { + return fmt.Errorf("policy entry for slot %d out of range", e.Key.Slot) + } + if e.Key.Kind == privpolicy.KindMeta { + if e.Value.Generation != p.Generation { + return fmt.Errorf("slot %d carries another generation", e.Key.Slot) + } + metas++ + } + } + if metas != p.Slots { + return fmt.Errorf("every slot needs one generation entry") + } + return nil +} + +// writePolicy checks the map layout against privilege_policy.h and writes the +// compiled entries. +func writePolicy(entries []privpolicy.KernelEntry) func(enforcement.Map) error { + return func(m enforcement.Map) error { + if m == nil || m.Type() != ebpf.Hash || m.KeySize() != privpolicy.KernelKeySize || m.ValueSize() != privpolicy.KernelValueSize || m.MaxEntries() != privpolicy.KernelMaxEntries || m.Flags()&unix.BPF_F_RDONLY_PROG == 0 { + return fmt.Errorf("invalid immutable privilege policy map layout") + } + for _, e := range entries { + if err := m.Put(e.Key, e.Value); err != nil { + return fmt.Errorf("writing policy: %w", err) + } + } + return nil + } +} + +type operator struct{} + +func New() operators.DataOperator { return &operator{} } +func init() { operators.RegisterDataOperator(New()) } +func (*operator) Name() string { return operatorName } +func (*operator) Priority() int { return 1 } +func (*operator) Init(*params.Params) error { return nil } +func (*operator) GlobalParams() api.Params { return nil } +func (*operator) InstanceParams() api.Params { return nil } + +func (*operator) InstantiateDataOperator(gc operators.GadgetContext, _ api.ParamValues) (operators.DataOperatorInstance, error) { + value, _ := gc.GetVar("config") + metadata, ok := value.(*viper.Viper) + if !ok || !metadata.GetBool(annotation) { + return nil, nil + } + value, _ = gc.GetVar(configKey) + config, supplied := value.(Config) + if !supplied { + return nil, fmt.Errorf("enforce_privilege requires native policy configuration; use runtime.WithConfig") + } + if err := validateConfig(config); err != nil { + return nil, err + } + b, err := enforcement.New(enforcement.Config{ + Name: operatorName, + Generation: config.Policy.Generation, + Slots: config.Policy.Slots, + InitialBindings: config.InitialBindings, + PolicyMap: "policy", + WritePolicy: writePolicy(config.Policy.Entries), + ArmMap: "armed_tasks", + }) + if err != nil { + return nil, err + } + gc.SetVar(bindingsKey, b) + return b, nil +} diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go new file mode 100644 index 0000000000..ae02b6a24e --- /dev/null +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go @@ -0,0 +1,65 @@ +// Copyright 2026 The Kubescape Authors +// SPDX-License-Identifier: Apache-2.0 + +package runtime + +import ( + "testing" + + "github.com/cilium/ebpf" + "golang.org/x/sys/unix" + + privpolicy "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/policy" +) + +type fakePolicyMap struct { + flags uint32 + puts int +} + +func (m *fakePolicyMap) Put(_, _ interface{}) error { m.puts++; return nil } +func (m *fakePolicyMap) Delete(interface{}) error { return nil } +func (m *fakePolicyMap) Type() ebpf.MapType { return ebpf.Hash } +func (m *fakePolicyMap) KeySize() uint32 { return privpolicy.KernelKeySize } +func (m *fakePolicyMap) ValueSize() uint32 { return privpolicy.KernelValueSize } +func (m *fakePolicyMap) MaxEntries() uint32 { return privpolicy.KernelMaxEntries } +func (m *fakePolicyMap) Flags() uint32 { return m.flags } +func (m *fakePolicyMap) Freeze() error { return nil } + +func compiled(t *testing.T) privpolicy.Compiled { + t.Helper() + d := privpolicy.Document{Version: 1, Policy: privpolicy.Policy{Mode: "enforce", Privilege: privpolicy.Section{Default: "allow", + Rules: []privpolicy.Rule{{Name: "r", Action: "deny", Target: privpolicy.Target{Syscall: []string{"mount"}}}}}}} + c, err := privpolicy.CompileKernel(d, 5) + if err != nil { + t.Fatal(err) + } + return c +} + +func TestWritePolicyChecksLayout(t *testing.T) { + c := compiled(t) + m := &fakePolicyMap{flags: unix.BPF_F_RDONLY_PROG} + if err := writePolicy(c.Entries)(m); err != nil || m.puts != len(c.Entries) { + t.Fatalf("write: %v puts=%d", err, m.puts) + } + if err := writePolicy(c.Entries)(&fakePolicyMap{}); err == nil { + t.Fatal("writable policy map must be rejected") + } +} + +func TestValidateConfig(t *testing.T) { + c := compiled(t) + if err := validateConfig(Config{Policy: c}); err != nil { + t.Fatal(err) + } + c.Generation = 9 + if err := validateConfig(Config{Policy: c}); err == nil { + t.Fatal("generation mismatch must fail") + } + c = compiled(t) + c.Slots = 2 + if err := validateConfig(Config{Policy: c}); err == nil { + t.Fatal("a slot without its generation entry must fail") + } +} From de8c00685180ae7bf4408b28675a657235b94915 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:18:02 +0200 Subject: [PATCH 02/34] fix(enforce_privilege): address review comments - each namespace flag of a call resolves on its own (bpf_loop), so an allow for one flag cannot carry another flag past a default deny - compat ptrace read-mode access is not judged - failed arming marks are counted; a deep tree arms instead of matching a wrong namespace - transient arming failures are retried and reported - a binding whose rollback failed stays tracked for Unbind - reject NUL in mountFsType and duplicate slot generation entries - the gadget build loop stops on the first failure Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- Makefile | 2 +- pkg/ebpf/gadgets/common/arming.h | 29 ++++--- .../gadgets/common/enforcement/arm_linux.go | 20 ++++- .../gadgets/common/enforcement/bindings.go | 5 +- pkg/ebpf/gadgets/common/verdict.h | 2 +- pkg/ebpf/gadgets/enforce_privilege/README.md | 9 +- .../enforce_privilege/policy/kernel_test.go | 1 + .../enforce_privilege/policy/policy.go | 4 +- .../gadgets/enforce_privilege/program.bpf.c | 84 +++++++++++++++---- .../enforce_privilege/runtime/runtime.go | 9 +- .../enforce_privilege/runtime/runtime_test.go | 4 + 11 files changed, 126 insertions(+), 43 deletions(-) diff --git a/Makefile b/Makefile index 8fea1ceb69..8c4c5e8ba0 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ docker-push: docker-build docker push $(IMAGE):$(TAG) gadgets: - $(foreach img,$(KUBESCAPE_GADGETS),$(MAKE) -C ./pkg/ebpf/gadgets/$(img) build IMAGE=$(img) TAG=latest;) + $(foreach img,$(KUBESCAPE_GADGETS),$(MAKE) -C ./pkg/ebpf/gadgets/$(img) build IMAGE=$(img) TAG=latest || exit 1;) $(foreach img,$(BUILT_GADGETS),$(MAKE) -C ./pkg/ebpf/gadgets/$(img) build IMAGE=ghcr.io/inspektor-gadget/gadget/$(img) TAG=$(VERSION);) $(foreach img,$(GADGETS),sudo ig image pull ghcr.io/inspektor-gadget/gadget/$(img):$(VERSION);) sudo ig image export $(foreach img,$(GADGETS) $(BUILT_GADGETS),ghcr.io/inspektor-gadget/gadget/$(img):$(VERSION)) $(foreach img,$(KUBESCAPE_GADGETS),$(img):latest) tracers.tar diff --git a/pkg/ebpf/gadgets/common/arming.h b/pkg/ebpf/gadgets/common/arming.h index 4dda642b59..1ea8677d68 100644 --- a/pkg/ebpf/gadgets/common/arming.h +++ b/pkg/ebpf/gadgets/common/arming.h @@ -53,12 +53,16 @@ static __always_inline bool arming_task_armed(struct task_struct *task) return a && *a; } -static __always_inline void arming_mark(struct task_struct *task) +// arming_mark returns false when the task storage could not be allocated (memory +// pressure): the task then stays unarmed, and the caller counts the failure. +static __always_inline bool arming_mark(struct task_struct *task) { __u32 *a = bpf_task_storage_get(&armed_tasks, task, 0, BPF_LOCAL_STORAGE_GET_F_CREATE); - if (a) - *a = 1; + if (!a) + return false; + *a = 1; + return true; } #define ARMING_MAX_ANCESTORS 64 @@ -90,28 +94,31 @@ static __always_inline __u64 arming_host_mntns(struct task_struct *task) } t = parent; } - // Tree deeper than the walk: do not cache a value that may be wrong. - return BPF_CORE_READ(t, nsproxy, mnt_ns, ns.inum); + // Tree deeper than the walk: no namespace inode is 0, so the task is + // treated as outside the runtime namespace (armed), and nothing is cached. + return 0; } // arming_on_exec runs after a successful exec (bprm_committed_creds). -static __always_inline void arming_on_exec(void) +// arming_on_exec returns false when a needed mark failed. +static __always_inline bool arming_on_exec(void) { struct task_struct *task = bpf_get_current_task_btf(); __u64 mntns = BPF_CORE_READ(task, nsproxy, mnt_ns, ns.inum); if (arming_task_armed(task)) - return; + return true; if (mntns == arming_host_mntns(task)) - return; - arming_mark(task); + return true; + return arming_mark(task); } // arming_on_fork runs in task_alloc: the child inherits the parent's mark. -static __always_inline void arming_on_fork(struct task_struct *child) +static __always_inline bool arming_on_fork(struct task_struct *child) { if (arming_task_armed(bpf_get_current_task_btf())) - arming_mark(child); + return arming_mark(child); + return true; } #endif /* __ENFORCEMENT_ARMING_H */ diff --git a/pkg/ebpf/gadgets/common/enforcement/arm_linux.go b/pkg/ebpf/gadgets/common/enforcement/arm_linux.go index fea9596044..ef8b94e899 100644 --- a/pkg/ebpf/gadgets/common/enforcement/arm_linux.go +++ b/pkg/ebpf/gadgets/common/enforcement/arm_linux.go @@ -85,23 +85,35 @@ func armThreads(armed Map, threads string, hostMntns uint64, seen map[int]bool) if err != nil || seen[tid] { continue } - seen[tid] = true - if mntns, err := mntnsOf(scanner.Text()); err != nil || mntns == hostMntns { + // A task is seen once it is armed or needs no arming; any other + // failure is returned, so the caller does not report it protected. + mntns, err := mntnsOf(scanner.Text()) + if errors.Is(err, fs.ErrNotExist) || (err == nil && mntns == hostMntns) { + seen[tid] = true // exited, or a runtime helper continue } + if err != nil { + return fmt.Errorf("reading mount namespace of task %d: %w", tid, err) + } fd, err := unix.PidfdOpen(tid, pidfdThread) if errors.Is(err, unix.EINVAL) { fd, err = unix.PidfdOpen(tid, 0) } - if err != nil { - // The task exited, or it is a non-leader thread on an older kernel. + if errors.Is(err, unix.ESRCH) || errors.Is(err, unix.EINVAL) { + // Exited, or a non-leader thread on a kernel without + // PIDFD_THREAD: the BPF rule judges it through its leader. + seen[tid] = true continue } + if err != nil { + return fmt.Errorf("opening task %d: %w", tid, err) + } err = armed.Put(uint32(fd), uint32(1)) unix.Close(fd) if err != nil && !errors.Is(err, unix.ESRCH) { return fmt.Errorf("arming task %d: %w", tid, err) } + seen[tid] = true } return scanner.Err() } diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings.go b/pkg/ebpf/gadgets/common/enforcement/bindings.go index bccc68e614..d5bad43220 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings.go @@ -218,7 +218,10 @@ func (b *Bindings) bind(c Binding) error { // A container that is gone needs no protection; a startup // replay skips it (see start). if derr := b.members.Delete(id.CgroupID); derr != nil { - err = errors.Join(err, derr) + // The kernel entry stays: keep it tracked so Unbind can + // remove it later. + b.active[id.CgroupID] = c + return errors.Join(err, derr) } return err } diff --git a/pkg/ebpf/gadgets/common/verdict.h b/pkg/ebpf/gadgets/common/verdict.h index ea88abdedc..39d267af0b 100644 --- a/pkg/ebpf/gadgets/common/verdict.h +++ b/pkg/ebpf/gadgets/common/verdict.h @@ -38,7 +38,7 @@ enum verdict_counter { VERDICT_COUNT_SUBMITTED = 1, VERDICT_COUNT_DROPPED = 2, VERDICT_COUNT_ALLOWED = 3, - VERDICT_COUNT_SECTION_ERROR = 4, + VERDICT_COUNT_SECTION_ERROR = 4, // privilege: failed arming marks VERDICT_COUNT_RATE_LIMITED = 5, VERDICT_COUNT_MAX = 6, }; diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index e7eba05e86..30faeccc06 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -13,10 +13,11 @@ One instance runs per node: combination share it. Limits per node: 1024 containers, 64 slots, 512 policy map entries per slot, 16384 in total. - Binding or unbinding a container is a live write to the `bindings` map. -- A combination the running instance does not have (a new or edited policy) - replaces the node instance: the candidate starts with every slot in use, is - ready with every container bound, and then the old instance stops. Both - enforce during the overlap. Unused slots are dropped at the next replacement. +- The policy map is frozen per instance. A combination the running instance + does not have (a new or edited policy) needs a new instance: the owner starts + one with every slot in use and every container bound, and then stops the old + one, so both enforce during the overlap (the sandbox policy controller does + this). - An owner can create a slot before a container starts (for example when its pod is scheduled to the node), so the container binds with a live write. - Several policies on one container merge as the SandboxPolicy design says: the diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go index a7ebc481f8..55a97c12a3 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go @@ -131,6 +131,7 @@ func TestValidationErrors(t *testing.T) { "cap prefix": " - {name: r, target: {capability: [CAP_SYS_ADMIN]}, action: deny}\n", "bad action": " - {name: r, target: {syscall: [mount]}, action: block}\n", "long fstype": " - {name: r, target: {mountFsType: [abcdefghijklmnop]}, action: deny}\n", + "NUL fstype": " - {name: r, target: {mountFsType: [\"pro\\0c\"]}, action: deny}\n", "duplicate name": " - {name: r, target: {syscall: [mount]}, action: deny}\n - {name: r, target: {syscall: [bpf]}, action: deny}\n", "unknown field": " - {name: r, target: {path: [/x]}, action: deny}\n", } { diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go b/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go index f105c652ae..b44849ade6 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go @@ -124,8 +124,8 @@ func (t Target) validate() error { } } for _, fs := range t.MountFSType { - if fs == "" || len(fs) >= KernelNameLen { - return fmt.Errorf("filesystem type %q must be 1-%d bytes", fs, KernelNameLen-1) + if fs == "" || len(fs) >= KernelNameLen || strings.ContainsRune(fs, 0) { + return fmt.Errorf("filesystem type %q must be 1-%d bytes without NUL", fs, KernelNameLen-1) } } return nil diff --git a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c index e731de0ab0..f1b04a7b6a 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c +++ b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c @@ -289,25 +289,72 @@ static __always_inline void consider_syscall(__u32 slot, enum priv_syscall sc, priv_consider(&key, m); } -// resolve_syscall resolves the syscall kind: rules on the syscall with any -// flag, rules on each CLONE_NEW* flag in flags, then the syscall default. +struct resolve_flags_ctx { + __u32 slot; + enum priv_syscall sc; + __u64 flags; + struct policy_match base; + struct policy_match out; +}; + +static __always_inline __u64 ns_flag_at(__u32 i) +{ + switch (i) { + case 0: + return CLONE_NEWNS; + case 1: + return CLONE_NEWCGROUP; + case 2: + return CLONE_NEWUTS; + case 3: + return CLONE_NEWIPC; + case 4: + return CLONE_NEWUSER; + case 5: + return CLONE_NEWPID; + case 6: + return CLONE_NEWNET; + case 7: + return CLONE_NEWTIME; + } + return 0; +} + +// resolve_flag is a bpf_loop callback: one namespace flag of the call. +static long resolve_flag(__u32 i, void *data) +{ + struct resolve_flags_ctx *c = data; + __u64 flag = ns_flag_at(i); + struct policy_match m; + + if (!flag) + return 1; + if (!(c->flags & flag)) + return 0; + m = c->base; + consider_syscall(c->slot, c->sc, flag, &m); + priv_finish_kind(c->slot, PRIV_KIND_SYSCALL, &m); + policy_merge(&c->out, &m); + return 0; +} + +// resolve_syscall resolves the syscall kind. Each namespace flag of a call +// resolves on its own (rules on the flag or on the syscall with any flag, else +// the default), so an allow for one flag never lets another flag of the same +// call pass. The per-flag work runs in bpf_loop to keep verification small. static __always_inline void resolve_syscall(__u32 slot, enum priv_syscall sc, __u64 flags, struct policy_match *out) { - static const __u64 ns_flags[] = { - CLONE_NEWNS, CLONE_NEWCGROUP, CLONE_NEWUTS, CLONE_NEWIPC, - CLONE_NEWUSER, CLONE_NEWPID, CLONE_NEWNET, CLONE_NEWTIME, - }; - struct policy_match m = {}; + struct resolve_flags_ctx c = { .slot = slot, .sc = sc, .flags = flags }; - consider_syscall(slot, sc, 0, &m); -#pragma unroll - for (int i = 0; i < sizeof(ns_flags) / sizeof(ns_flags[0]); i++) { - if (flags & ns_flags[i]) - consider_syscall(slot, sc, ns_flags[i], &m); + consider_syscall(slot, sc, 0, &c.base); + if (!(flags & CLONE_NS_MASK)) { + priv_finish_kind(slot, PRIV_KIND_SYSCALL, &c.base); + policy_merge(out, &c.base); + return; } - priv_finish_kind(slot, PRIV_KIND_SYSCALL, &m); - policy_merge(out, &m); + bpf_loop(8, resolve_flag, &c, 0); + policy_merge(out, &c.out); } static __always_inline void resolve_capability(__u32 slot, int cap, @@ -416,7 +463,8 @@ static __always_inline int check_syscall(void *ctx, enum priv_hook hook, SEC("lsm/bprm_committed_creds") int BPF_PROG(ig_priv_exec) { - arming_on_exec(); + if (!arming_on_exec()) + verdict_count(VERDICT_COUNT_SECTION_ERROR); return 0; } @@ -462,7 +510,8 @@ static __always_inline int do_task_alloc(void *ctx, struct task_struct *task, enum priv_syscall sc; __u64 flags; - arming_on_fork(task); + if (!arming_on_fork(task)) + verdict_count(VERDICT_COUNT_SECTION_ERROR); // userns_create judges CLONE_NEWUSER. flags = clone_flags & CLONE_NS_MASK & ~CLONE_NEWUSER; @@ -696,6 +745,9 @@ static __always_inline int do_ptrace_access_check(void *ctx, unsigned int mode) __u32 slot = 0; if (current_is_compat()) { + // Read-mode /proc access (ps, top) is never ptrace. + if (!(mode & PTRACE_MODE_ATTACH)) + return 0; // Unknown syscall: apply all three ptrace-family syscalls. header = priv_scope(&cgid, &slot); if (!header) diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go index 8561476cf3..df8501e046 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go @@ -57,7 +57,7 @@ func validateConfig(config Config) error { if p.Generation == 0 || p.Slots > privpolicy.KernelMaxSlots || len(p.Entries) > privpolicy.KernelMaxEntries { return fmt.Errorf("invalid compiled privilege policy") } - metas := uint32(0) + metas := map[uint32]bool{} for _, e := range p.Entries { if e.Key.Slot >= p.Slots { return fmt.Errorf("policy entry for slot %d out of range", e.Key.Slot) @@ -66,10 +66,13 @@ func validateConfig(config Config) error { if e.Value.Generation != p.Generation { return fmt.Errorf("slot %d carries another generation", e.Key.Slot) } - metas++ + if metas[e.Key.Slot] { + return fmt.Errorf("slot %d has two generation entries", e.Key.Slot) + } + metas[e.Key.Slot] = true } } - if metas != p.Slots { + if uint32(len(metas)) != p.Slots { return fmt.Errorf("every slot needs one generation entry") } return nil diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go index ae02b6a24e..97a251a2c7 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go @@ -62,4 +62,8 @@ func TestValidateConfig(t *testing.T) { if err := validateConfig(Config{Policy: c}); err == nil { t.Fatal("a slot without its generation entry must fail") } + c.Entries = append(c.Entries, c.Entries[0]) // a second slot-0 entry, no slot 1 + if err := validateConfig(Config{Policy: c}); err == nil { + t.Fatal("duplicate generation entries must fail") + } } From 89fc29406425a7fe7d2be1887da7be57fbd4abc7 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:20:59 +0200 Subject: [PATCH 03/34] docs(enforce_privilege): document the known limits Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/enforce_privilege/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index 30faeccc06..8cbaa976c8 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -94,6 +94,17 @@ spec: candidate syscall rule applies. - **Deny** returns `-EPERM` from the LSM hook, before the side effect. +## Known limits + +- **Failed arming mark.** If the kernel cannot allocate task storage (memory + pressure), the task stays unarmed and is not judged until its next exec. The + failure is counted in `counters` slot 4. A later check cannot tell such a + task apart from the runtime's own setup, so it is not retried. +- **No-audit capability checks** (`CAP_OPT_NOAUDIT`) are not judged. They are + kernel probes (for example the overcommit check); judging them would deny + normal work in every bound container. One of them grants write access to + `/proc/sys/kernel/ns_last_pid`, which the privilege section does not cover. + ## Maps and events | Map | Content | From 931392e1a1b4ae1a84e2bb7aa6c12efa31d03f54 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:22:41 +0200 Subject: [PATCH 04/34] docs(enforce_privilege): correct the no-audit limit Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/enforce_privilege/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index 8cbaa976c8..b474cdf23e 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -100,10 +100,11 @@ spec: pressure), the task stays unarmed and is not judged until its next exec. The failure is counted in `counters` slot 4. A later check cannot tell such a task apart from the runtime's own setup, so it is not retried. -- **No-audit capability checks** (`CAP_OPT_NOAUDIT`) are not judged. They are - kernel probes (for example the overcommit check); judging them would deny - normal work in every bound container. One of them grants write access to - `/proc/sys/kernel/ns_last_pid`, which the privilege section does not cover. +- **No-audit capability checks** (`CAP_OPT_NOAUDIT`) are excluded from + capability enforcement. Most are kernel probes (for example the overcommit + check), and judging them would deny normal work in every bound container. Some + also grant access (in current kernels, writing the pid namespace's `pid_max`), + so `capability` rules do not cover every `CAP_SYS_ADMIN` use. ## Maps and events From ca6857cebf63efa0acc682f7d7441a8bf19068f8 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 01:18:02 +0200 Subject: [PATCH 05/34] fix(enforcement): report cgroup listing errors other than a removed cgroup Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/common/enforcement/arm_linux.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/ebpf/gadgets/common/enforcement/arm_linux.go b/pkg/ebpf/gadgets/common/enforcement/arm_linux.go index ef8b94e899..699fb6d4ed 100644 --- a/pkg/ebpf/gadgets/common/enforcement/arm_linux.go +++ b/pkg/ebpf/gadgets/common/enforcement/arm_linux.go @@ -51,7 +51,10 @@ func ArmExisting(armed Map, cgroupPath string) error { } return fmt.Errorf("listing container tasks: %w", err) } - return nil // a child cgroup removed during the walk + if errors.Is(err, fs.ErrNotExist) { + return nil // a child cgroup removed during the walk + } + return fmt.Errorf("listing container tasks: %w", err) } if !d.IsDir() { return nil @@ -75,9 +78,12 @@ const maxArmingPasses = 8 func armThreads(armed Map, threads string, hostMntns uint64, seen map[int]bool) error { f, err := os.Open(threads) - if err != nil { + if errors.Is(err, fs.ErrNotExist) { return nil // the cgroup was removed } + if err != nil { + return fmt.Errorf("listing container tasks: %w", err) + } defer f.Close() scanner := bufio.NewScanner(f) for scanner.Scan() { From d06212fd92c91184c5debb1c75511e8f7f4a7889 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 10:03:22 +0200 Subject: [PATCH 06/34] refactor(enforce_privilege): make the gadget and common library portable - the gadget has its own policy document with Parse and an example file; the SandboxPolicy translation moves to the controller connection (#1068) - neutral names in the gadget, BPF, gadget.yaml and README - a guard test fails on imports outside Inspektor Gadget and the gadget folders, and on host-product names or outside includes - an opt-in test runs the gadget with Inspektor Gadget alone Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/common/arming.h | 2 +- .../common/enforcement/portability_test.go | 116 +++++++++++++++ pkg/ebpf/gadgets/enforce_privilege/README.md | 75 ++++++---- .../enforce_privilege/examples/policy.yaml | 28 ++++ .../gadgets/enforce_privilege/gadget.yaml | 2 +- .../gadgets/enforce_privilege/policy/api.go | 24 --- .../enforce_privilege/policy/kernel.go | 8 +- .../enforce_privilege/policy/kernel_test.go | 38 +++-- .../enforce_privilege/policy/policy.go | 36 ++++- .../gadgets/enforce_privilege/program.bpf.c | 3 +- .../runtime/runtime_kernel_test.go | 139 ++++++++++++++++++ 11 files changed, 396 insertions(+), 75 deletions(-) create mode 100644 pkg/ebpf/gadgets/common/enforcement/portability_test.go create mode 100644 pkg/ebpf/gadgets/enforce_privilege/examples/policy.yaml delete mode 100644 pkg/ebpf/gadgets/enforce_privilege/policy/api.go create mode 100644 pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go diff --git a/pkg/ebpf/gadgets/common/arming.h b/pkg/ebpf/gadgets/common/arming.h index 1ea8677d68..d845e3d08a 100644 --- a/pkg/ebpf/gadgets/common/arming.h +++ b/pkg/ebpf/gadgets/common/arming.h @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright 2026 The Kubescape Authors -// Arming rule shared by the SandboxPolicy prevention gadgets. +// Arming rule shared by the BPF LSM enforcement gadgets. // // A cgroup-scoped enforcer also sees the container runtime's own setup inside // the container cgroup (runc init creates namespaces, mounts, pivots root and, diff --git a/pkg/ebpf/gadgets/common/enforcement/portability_test.go b/pkg/ebpf/gadgets/common/enforcement/portability_test.go new file mode 100644 index 0000000000..7b5dee9477 --- /dev/null +++ b/pkg/ebpf/gadgets/common/enforcement/portability_test.go @@ -0,0 +1,116 @@ +// Copyright 2026 The Kubescape Authors +// SPDX-License-Identifier: Apache-2.0 + +package enforcement + +import ( + "go/parser" + "go/token" + "io/fs" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" +) + +// The gadget folders must move into Inspektor Gadget with only an import-path +// rewrite. So they may import the standard library, Inspektor Gadget (and its +// enforce_exec types), the few libraries Inspektor Gadget itself uses, and each +// other — nothing else from the host repository and no Kubernetes or policy-CRD +// concepts. +var gadgetFolders = []string{"..", "../../enforce_privilege"} + +const ownModule = "github.com/kubescape/node-agent/pkg/ebpf/gadgets/" + +var allowedImports = []string{ + "github.com/inspektor-gadget/inspektor-gadget/", + "github.com/cilium/ebpf", + "github.com/spf13/viper", + "golang.org/x/sys/", + "gopkg.in/yaml.v3", + "oras.land/oras-go/v2", // Inspektor Gadget's OCI store, used by a test + ownModule + "common/", + ownModule + "enforce_privilege/", +} + +// Names of the host product and its policy API, which the gadgets must not use. +var forbiddenWords = regexp.MustCompile(`(?i)sandbox[ _-]?polic|kubescape\.io|\barmo(sec)?\b|node-agent|kubescape/storage|\bkubernetes\b|\bpods?\b`) + +// A BPF include must stay inside the gadget folders: "x.h" or "../common/x.h". +var badInclude = regexp.MustCompile(`#include\s+"(\.\./){2,}|#include\s+"\.\./[^c]`) + +func walkGadgetFiles(t *testing.T, fn func(path string, data []byte)) { + t.Helper() + for _, root := range gadgetFolders { + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if d.Name() == "lab" { // lab harness, not part of the gadget + return filepath.SkipDir + } + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + fn(path, data) + return nil + }) + if err != nil { + t.Fatal(err) + } + } +} + +func TestGadgetImportsArePortable(t *testing.T) { + walkGadgetFiles(t, func(path string, data []byte) { + if !strings.HasSuffix(path, ".go") { + return + } + f, err := parser.ParseFile(token.NewFileSet(), path, data, parser.ImportsOnly) + if err != nil { + t.Fatal(err) + } + for _, imp := range f.Imports { + p, _ := strconv.Unquote(imp.Path.Value) + if !strings.Contains(strings.SplitN(p, "/", 2)[0], ".") { + continue // standard library + } + ok := false + for _, allowed := range allowedImports { + if p == strings.TrimSuffix(allowed, "/") || strings.HasPrefix(p, allowed) { + ok = true + } + } + if !ok { + t.Errorf("%s imports %s, which a portable gadget must not use", path, p) + } + } + }) +} + +func TestGadgetFilesUseNeutralNames(t *testing.T) { + walkGadgetFiles(t, func(path string, data []byte) { + if filepath.Base(path) == "portability_test.go" && filepath.Base(filepath.Dir(path)) == "enforcement" { + return // this guard lists the forbidden names itself + } + for i, line := range strings.Split(string(data), "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, `"`+ownModule) || strings.Contains(trimmed, ` "`+ownModule) || + strings.HasPrefix(trimmed, "// Copyright") || strings.HasPrefix(trimmed, "# Copyright") { + continue // import paths are rewritten on a move; license headers stay + } + if badInclude.MatchString(line) { + t.Errorf("%s:%d includes a header outside the gadget folders", path, i+1) + } + if m := forbiddenWords.FindString(line); m != "" { + t.Errorf("%s:%d uses %q: gadget files must use neutral names", path, i+1, m) + } + } + }) +} diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index b474cdf23e..bb1e844fa0 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -1,35 +1,27 @@ # enforce_privilege -Container privilege enforcement with BPF LSM, no seccomp. It enforces the -`privilege` section of a sandbox policy: namespace creation and joining, mounts, -`pivot_root`/`chroot`, capabilities, ptrace, kernel modules, `bpf`, -`perf_event_open`, keys, `io_uring_setup`, `userfaultfd`, `mknod`, -`setuid`/`setgid` and `capset`. +Container privilege enforcement with BPF LSM, no seccomp. It enforces a +privilege policy: namespace creation and joining, mounts, `pivot_root`/`chroot`, +capabilities, ptrace, kernel modules, `bpf`, `perf_event_open`, keys, +`io_uring_setup`, `userfaultfd`, `mknod`, `setuid`/`setgid` and `capset`. The gadget is built on the [common enforcement library](../common/README.md). -One instance runs per node: +It depends only on Inspektor Gadget and that library. One instance runs per +node: -- A **slot** is one combination of privilege policies; containers with the same +- A **slot** is one combination of policy documents; containers with the same combination share it. Limits per node: 1024 containers, 64 slots, 512 policy map entries per slot, 16384 in total. - Binding or unbinding a container is a live write to the `bindings` map. - The policy map is frozen per instance. A combination the running instance does not have (a new or edited policy) needs a new instance: the owner starts one with every slot in use and every container bound, and then stops the old - one, so both enforce during the overlap (the sandbox policy controller does - this). -- An owner can create a slot before a container starts (for example when its - pod is scheduled to the node), so the container binds with a live write. -- Several policies on one container merge as the SandboxPolicy design says: the - union of rules (explicit deny > allow > default, each rule with its own name, - origin and mode); a default is deny if any policy denies, with the strictest - mode among the policies that agree. One addition to the design: a monitoring - (audit) deny rule never weakens another policy's enforce deny, including its - enforce default deny. An explicit allow still beats another policy's default - deny. - -`policy/` compiles the `privilege` section of one or more policies; -`runtime/` is the native owner a controller drives. + one, so both enforce during the overlap. +- An owner can create a slot before a container starts, so the container binds + with a live write. + +`policy/` parses and compiles policy documents; `runtime/` is the native +operator an owner drives with `runtime.WithConfig` and `runtime.GetBindings`. Requires cgroup v2, kernel BTF and BPF LSM enabled (`bpf` in `/sys/kernel/security/lsm`). Linux 6.1 or later (the `userns_create` hook; @@ -39,38 +31,46 @@ README). ## Policy -The `privilege` section of a SandboxPolicy, for example: +A policy document (YAML or JSON), parsed with `policy.Parse`. Full example: +[examples/policy.yaml](examples/policy.yaml). ```yaml -spec: +version: 1 +name: example # qualifies rule names in events +policy: mode: enforce # audit | enforce | off privilege: - default: allow # applies to syscall and mountFsType targets + default: allow # default for syscall and mountFsType targets rules: - name: no-new-namespaces target: {syscall: [unshare, clone, clone3, setns], flags: [CLONE_NEWUSER, CLONE_NEWNS]} action: deny - origin: catalog/G1 - name: no-proc-mount target: {mountFsType: [proc, sysfs]} action: deny - - name: audit-sys-admin + - name: watch-sys-admin target: {capability: [SYS_ADMIN]} action: deny - mode: audit + mode: audit # a rule may override the document mode ``` - A rule targets exactly one kind: `syscall` (optionally with `flags` for `unshare`, `setns`, `clone`, `clone3`), `capability` (short names such as `SYS_ADMIN`) or `mountFsType`. - Resolution: an explicit deny beats an allow, an allow beats the default, - independent of rule order. Each rule has its own mode. + independent of rule order. Each rule has its own mode. Each namespace flag of + a call resolves on its own. - Capability checks have no default; only explicit capability rules apply. - A `default: deny` also denies `setuid`/`setgid` (package managers drop privileges), implicit kernel module loads (judged as `init_module`, for example an `iptables` match module) and new superblocks of any filesystem type. Allow what the workload needs. -- `actor`, `except` and `pathClass` are not supported yet. +- **Several documents in one slot** combine so that the most limiting result + wins: the union of rules (explicit deny > allow > default, each rule with its + own name, origin and mode); a default is deny if any document denies, with the + strictest mode among the documents that agree; a monitoring (audit) deny rule + never weakens another document's enforce deny, including its enforce default + deny; an explicit allow still beats another document's default deny. ## How it decides @@ -106,6 +106,19 @@ spec: also grant access (in current kernels, writing the pid namespace's `pid_max`), so `capability` rules do not cover every `CAP_SYS_ADMIN` use. +## Using it outside this repository + +- The gadget and `../common` depend only on Inspektor Gadget, the libraries it + uses, and the `enforce_exec` types of the Inspektor Gadget fork. A guard test + (`common/enforcement/portability_test.go`) keeps it that way. +- A Go program drives it: `runtime.WithConfig` with a compiled policy, then + `runtime.GetBindings` to bind containers (see `runtime/runtime_kernel_test.go`). + There is no `ig` command for it yet; `ig run` alone fails because the policy + comes from the owner, not from gadget parameters. +- To move it into Inspektor Gadget: rewrite the import paths, drop the + per-gadget `Makefile` (its `REPO_ROOT` is this repository's depth) and add the + gadget to the central gadget list, which already sets `IG_SOURCE_PATH`. + ## Maps and events | Map | Content | @@ -115,7 +128,9 @@ spec: The `verdict` datasource shares `generation`, `cgroup`, `deny`, `mode` and `host_pid` with enforce_exec and adds `rule_id`, `origin_id`, `is_default`, -`comm`, `hook`, `syscall`, `flags`, `capability`, `fstype` and `slot`. Rule and origin +`comm`, `hook`, `syscall`, `flags`, `capability`, `fstype` and `slot`. When a +document's default deny strengthens another document's audit rule on the same +key, the event names the rule `default` with `is_default` false. Rule and origin names are in the owner's compiled policy (`RuleNames`, `Origins`). Reporting is limited to 100 events per CPU per second; enforcement is never throttled. diff --git a/pkg/ebpf/gadgets/enforce_privilege/examples/policy.yaml b/pkg/ebpf/gadgets/enforce_privilege/examples/policy.yaml new file mode 100644 index 0000000000..fff38d6dd3 --- /dev/null +++ b/pkg/ebpf/gadgets/enforce_privilege/examples/policy.yaml @@ -0,0 +1,28 @@ +# enforce_privilege policy document. One document per policy; the compiler can +# merge several documents into one slot (most limiting wins). +version: 1 +name: example +policy: + mode: enforce # audit | enforce | off + privilege: + default: allow # default for syscall and mountFsType targets + rules: + - name: no-new-namespaces + target: + syscall: [unshare, clone, clone3, setns] + flags: [CLONE_NEWUSER, CLONE_NEWNS] + action: deny + origin: example/namespaces + - name: no-proc-mount + target: + mountFsType: [proc, sysfs] + action: deny + - name: no-kernel-modules + target: + syscall: [init_module, finit_module] + action: deny + - name: watch-sys-admin + target: + capability: [SYS_ADMIN] + action: deny + mode: audit diff --git a/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml b/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml index 9024f4ffff..11eb6602bb 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml +++ b/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml @@ -1,6 +1,6 @@ name: enforce_privilege behavior: mutator -description: "Enforce SandboxPolicy privilege rules (capabilities, namespaces, mounts, ptrace, modules) for a live set of containers (BPF LSM)" +description: "Enforce privilege rules (capabilities, namespaces, mounts, ptrace, modules) for a live set of containers (BPF LSM)" annotations: enable-containers-datasource: "true" enforce-privilege: "true" diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/api.go b/pkg/ebpf/gadgets/enforce_privilege/policy/api.go deleted file mode 100644 index 0b6294b907..0000000000 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/api.go +++ /dev/null @@ -1,24 +0,0 @@ -package privpolicy - -import ( - "fmt" - - api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" -) - -// FromAPI converts a SandboxPolicy privilege section into a validated Document. -// Actor and path predicates are not supported yet and are rejected. -func FromAPI(name string, mode api.PolicyMode, section api.PrivilegeSection) (Document, error) { - doc := Document{Name: name, Version: 1, Policy: Policy{Mode: string(mode), Privilege: Section{Default: string(section.Default)}}} - for _, r := range section.Rules { - t := r.Target - if r.Actor != nil || len(t.Path) > 0 || len(t.PathClass) > 0 || t.FromImage != nil || len(t.FSType) > 0 { - return doc, fmt.Errorf("rule %q: actor and path targets are unsupported in the privilege section", r.Name) - } - doc.Policy.Privilege.Rules = append(doc.Policy.Privilege.Rules, Rule{ - Name: r.Name, Action: string(r.Action), Mode: string(r.Mode), Origin: r.Origin, - Target: Target{Syscall: t.Syscall, Flags: t.Flags, Capability: t.Capability, MountFSType: t.MountFSType}, - }) - } - return doc, doc.Validate() -} diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go index ff7d7b121e..60db0f2e7e 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go @@ -119,12 +119,14 @@ func CompileKernel(d Document, generation uint64) (Compiled, error) { } // CompileSlots compiles one node instance. Each slot holds one combination of -// policies, merged as the CRD design says for several policies matching one pod: +// documents, combined so that the most limiting result wins: // - rules are the union, each with its own name, origin and mode; the compiler // folds every rule on one key into the most limiting rank, so an explicit // deny beats an allow and an allow beats a default; -// - a default is deny if any policy's default is deny, else allow; its mode is -// the strictest among the policies whose default equals the merged one. +// - a default is deny if any document's default is deny, else allow; its mode +// is the strictest among the documents whose default equals the merged one; +// - an audit deny rule never weakens another document's enforce default deny +// (see the raise below), while an explicit allow still beats it. // // The default covers the syscall and mountFsType kinds only. Capability checks // have no default: only explicit capability rules apply, because capable() runs diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go index 55a97c12a3..d153fd10db 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go @@ -5,22 +5,12 @@ package privpolicy import ( "encoding/binary" + "os" "strings" "testing" - - "gopkg.in/yaml.v3" ) -// parse decodes a test policy with unknown fields rejected, then validates it. -func parse(y string) (Document, error) { - var d Document - dec := yaml.NewDecoder(strings.NewReader(y)) - dec.KnownFields(true) - if err := dec.Decode(&d); err != nil { - return d, err - } - return d, d.Validate() -} +func parse(y string) (Document, error) { return Parse(strings.NewReader(y)) } func mustParse(t *testing.T, y string) Document { t.Helper() @@ -248,3 +238,27 @@ func TestAuditDenyRaiseRespectsFlagKeys(t *testing.T) { t.Fatalf("unshare --net: rank %d, want enforce from B's default", r) } } + +func TestParse(t *testing.T) { + f, err := os.Open("../examples/policy.yaml") + if err != nil { + t.Fatal(err) + } + defer f.Close() + d, err := Parse(f) + if err != nil { + t.Fatalf("example: %v", err) + } + if _, err := CompileKernel(d, 1); err != nil { + t.Fatalf("example compile: %v", err) + } + for name, in := range map[string]string{ + "unknown field": "version: 1\npolicy: {mode: enforce, privilege: {default: allow}}\nbinding: {}\n", + "two documents": "version: 1\npolicy: {mode: enforce, privilege: {default: allow}}\n---\nversion: 1\n", + "too large": "version: 1\n#" + strings.Repeat("x", MaxBytes) + "\n", + } { + if _, err := parse(in); err == nil { + t.Errorf("%s: expected error", name) + } + } +} diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go b/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go index b44849ade6..d44d2dd39c 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go @@ -1,16 +1,24 @@ // Copyright 2026 The Kubescape Authors // SPDX-License-Identifier: Apache-2.0 -// Package privpolicy parses and compiles the privilege section of a sandbox -// policy into the enforce_privilege map entries. +// Package privpolicy defines the enforce_privilege policy document (YAML or +// JSON, see ../examples/policy.yaml), parses it, and compiles one or more +// documents into the gadget's map entries. package privpolicy import ( + "bytes" "fmt" + "io" "strings" + + "gopkg.in/yaml.v3" ) -const MaxRules = 128 +const ( + MaxBytes = 64 * 1024 + MaxRules = 128 +) type Document struct { // Name qualifies rule names in events when several policies merge. @@ -45,6 +53,28 @@ type Target struct { MountFSType []string `yaml:"mountFsType,omitempty" json:"mountFsType,omitempty"` } +// Parse reads one policy document (YAML or JSON) with unknown fields rejected. +func Parse(r io.Reader) (Document, error) { + var doc Document + data, err := io.ReadAll(io.LimitReader(r, MaxBytes+1)) + if err != nil { + return doc, err + } + if len(data) > MaxBytes { + return doc, fmt.Errorf("policy exceeds %d bytes", MaxBytes) + } + dec := yaml.NewDecoder(bytes.NewReader(data)) + dec.KnownFields(true) + if err := dec.Decode(&doc); err != nil { + return doc, fmt.Errorf("parsing policy: %w", err) + } + var extra any + if err := dec.Decode(&extra); err != io.EOF { + return doc, fmt.Errorf("parsing policy: one document per file") + } + return doc, doc.Validate() +} + func validMode(m string, allowEmpty bool) bool { return m == "audit" || m == "enforce" || (allowEmpty && m == "") } diff --git a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c index f1b04a7b6a..13d0ae97a4 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c +++ b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c @@ -1,7 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright 2026 The Kubescape Authors -// enforce_privilege: enforce the privilege section of a SandboxPolicy with BPF +// enforce_privilege: enforce a privilege policy (namespaces, mounts, +// capabilities, ptrace, modules and more) for a set of containers with BPF // LSM. The program is fixed; the policy is map data (privilege_policy.h), // frozen per instance; a rule change starts a replacement instance. Audit and // enforce share one code path: the rank of the deciding entry carries the mode. diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go new file mode 100644 index 0000000000..134d09f458 --- /dev/null +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go @@ -0,0 +1,139 @@ +//go:build linux && privilege_kernel + +// Copyright 2026 The Kubescape Authors +// SPDX-License-Identifier: Apache-2.0 + +package runtime + +// Opt-in test of the gadget with Inspektor Gadget alone: it parses the example +// policy file, starts the gadget from an OCI archive and binds one Docker +// container through this package's operator. Requires root, BPF LSM and a +// disposable container started with seccomp=unconfined and --cap-add SYS_ADMIN: +// +// PRIVILEGE_KERNEL_TEST=1 +// PRIVILEGE_IMAGE_ARCHIVE OCI archive from `ig image export` +// PRIVILEGE_IMAGE image name in the archive +// PRIVILEGE_CONTAINER_ID full Docker container ID +// PRIVILEGE_CGROUP_PATH its cgroup v2 path (relative to /sys/fs/cgroup) + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strconv" + "sync/atomic" + "testing" + "time" + + "github.com/inspektor-gadget/inspektor-gadget/pkg/datasource" + gadgetcontext "github.com/inspektor-gadget/inspektor-gadget/pkg/gadget-context" + "github.com/inspektor-gadget/inspektor-gadget/pkg/gadget-service/api" + apihelpers "github.com/inspektor-gadget/inspektor-gadget/pkg/gadget-service/api-helpers" + instancemanager "github.com/inspektor-gadget/inspektor-gadget/pkg/gadget-service/instance-manager" + "github.com/inspektor-gadget/inspektor-gadget/pkg/operators" + _ "github.com/inspektor-gadget/inspektor-gadget/pkg/operators/ebpf" + ocihandler "github.com/inspektor-gadget/inspektor-gadget/pkg/operators/oci-handler" + "github.com/inspektor-gadget/inspektor-gadget/pkg/operators/simple" + "github.com/inspektor-gadget/inspektor-gadget/pkg/runtime/local" + "golang.org/x/sys/unix" + orasoci "oras.land/oras-go/v2/content/oci" + + "github.com/kubescape/node-agent/pkg/ebpf/gadgets/common/enforcement" + privpolicy "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/policy" +) + +func TestGadgetWithInspektorGadgetAlone(t *testing.T) { + if os.Getenv("PRIVILEGE_KERNEL_TEST") != "1" { + t.Skip("explicit disposable kernel fixture opt-in required") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + f, err := os.Open("../examples/policy.yaml") + if err != nil { + t.Fatal(err) + } + doc, err := privpolicy.Parse(f) + f.Close() + if err != nil { + t.Fatal(err) + } + compiled, err := privpolicy.CompileKernel(doc, 1) + if err != nil { + t.Fatal(err) + } + id, cgroupPath := os.Getenv("PRIVILEGE_CONTAINER_ID"), os.Getenv("PRIVILEGE_CGROUP_PATH") + var st unix.Stat_t + if err := unix.Stat(filepath.Join("/sys/fs/cgroup", cgroupPath), &st); err != nil { + t.Fatal(err) + } + container := ContainerIdentity{Runtime: "docker", ID: id, CgroupID: st.Ino, CgroupPath: cgroupPath} + + store, err := orasoci.NewFromTar(ctx, os.Getenv("PRIVILEGE_IMAGE_ARCHIVE")) + if err != nil { + t.Fatal(err) + } + rt := local.New() + if err := rt.Init(nil); err != nil { + t.Fatal(err) + } + defer rt.Close() + manager, err := instancemanager.New(rt) + if err != nil { + t.Fatal(err) + } + handler := ocihandler.New() + params := apihelpers.ToParamDescs(handler.GlobalParams()).ToParams() + if err := params.Set("allow-enforcement", "true"); err != nil { + t.Fatal(err) + } + if err := handler.Init(params); err != nil { + t.Fatal(err) + } + var verdicts atomic.Int32 + var bindings *enforcement.Bindings + events := simple.New("privilege-test-verdicts", simple.WithPriority(100), simple.OnInit(func(gc operators.GadgetContext) error { + var err error + if bindings, err = GetBindings(gc); err != nil { + return err + } + return gc.GetDataSources()["verdict"].Subscribe(func(datasource.DataSource, datasource.Data) error { + verdicts.Add(1) + return nil + }, 100) + })) + runtimeMntns, err := enforcement.RuntimeMountNamespace() + if err != nil { + t.Fatal(err) + } + desired := &api.GadgetInstance{Id: "privilege-kernel-test", GadgetConfig: &api.GadgetRunRequest{Version: api.VersionGadgetRunProtocol, ImageName: os.Getenv("PRIVILEGE_IMAGE"), + ParamValues: map[string]string{"operator.oci.ebpf.host-mntns-id": strconv.FormatUint(runtimeMntns, 10)}}} + if err := manager.StartGadget(ctx, desired, gadgetcontext.WithOrasReadonlyTarget(store), + gadgetcontext.WithAdditionalDataOperators(handler, New(), events), + WithConfig(Config{Policy: compiled})); err != nil { + t.Fatal(err) + } + defer manager.StopGadget(context.Background(), desired.Id) + + unshare := func() error { return exec.CommandContext(ctx, "docker", "exec", id, "unshare", "-m", "true").Run() } + if err := unshare(); err != nil { + t.Fatalf("unbound container must not be judged: %v", err) + } + if err := bindings.Bind(enforcement.Binding{Container: container}); err != nil { + t.Fatal(err) + } + if unshare() == nil { + t.Fatal("the example policy denies unshare(CLONE_NEWNS) in a bound container") + } + if err := bindings.Unbind(container); err != nil { + t.Fatal(err) + } + if err := unshare(); err != nil { + t.Fatalf("unbound again must be allowed: %v", err) + } + time.Sleep(500 * time.Millisecond) + if verdicts.Load() == 0 { + t.Fatal("no verdict event for the deny") + } +} From 5d5a8459e8eacbbdfd80d05ffb837112887b63a8 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 12:03:53 +0200 Subject: [PATCH 07/34] docs(enforce_privilege): mountFsType does not cover bind mounts Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/enforce_privilege/README.md | 3 +++ pkg/ebpf/gadgets/enforce_privilege/examples/policy.yaml | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index bb1e844fa0..920bdb2658 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -61,6 +61,9 @@ policy: independent of rule order. Each rule has its own mode. Each namespace flag of a call resolves on its own. - Capability checks have no default; only explicit capability rules apply. +- `mountFsType` judges the creation of a new filesystem instance (a new + superblock, from `mount(2)` or the new mount API). Bind mounts and remounts + of an existing one are judged only by the `mount` syscall rule. - A `default: deny` also denies `setuid`/`setgid` (package managers drop privileges), implicit kernel module loads (judged as `init_module`, for example an `iptables` match module) and new superblocks of any filesystem diff --git a/pkg/ebpf/gadgets/enforce_privilege/examples/policy.yaml b/pkg/ebpf/gadgets/enforce_privilege/examples/policy.yaml index fff38d6dd3..8be030b89a 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/examples/policy.yaml +++ b/pkg/ebpf/gadgets/enforce_privilege/examples/policy.yaml @@ -13,8 +13,8 @@ policy: flags: [CLONE_NEWUSER, CLONE_NEWNS] action: deny origin: example/namespaces - - name: no-proc-mount - target: + - name: no-proc-mount # new proc/sysfs mounts only; bind mounts + target: # are judged by the `mount` syscall rule mountFsType: [proc, sysfs] action: deny - name: no-kernel-modules From 630299d64cf785fb1eea8ce009053095778fdab2 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:05:46 +0200 Subject: [PATCH 08/34] test(enforcement): only ../common/ includes may leave the gadget folder Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- .../common/enforcement/portability_test.go | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/pkg/ebpf/gadgets/common/enforcement/portability_test.go b/pkg/ebpf/gadgets/common/enforcement/portability_test.go index 7b5dee9477..2a0e9a17c9 100644 --- a/pkg/ebpf/gadgets/common/enforcement/portability_test.go +++ b/pkg/ebpf/gadgets/common/enforcement/portability_test.go @@ -38,8 +38,18 @@ var allowedImports = []string{ // Names of the host product and its policy API, which the gadgets must not use. var forbiddenWords = regexp.MustCompile(`(?i)sandbox[ _-]?polic|kubescape\.io|\barmo(sec)?\b|node-agent|kubescape/storage|\bkubernetes\b|\bpods?\b`) -// A BPF include must stay inside the gadget folders: "x.h" or "../common/x.h". -var badInclude = regexp.MustCompile(`#include\s+"(\.\./){2,}|#include\s+"\.\./[^c]`) +// A quoted BPF include must stay inside the gadget folders: "x.h" or +// "../common/x.h" (checked by includeOutside). +var quotedInclude = regexp.MustCompile(`#include\s+"([^"]*)"`) + +func includeOutside(line string) bool { + m := quotedInclude.FindStringSubmatch(line) + if m == nil || !strings.HasPrefix(m[1], "../") { + return false + } + rest, ok := strings.CutPrefix(m[1], "../common/") + return !ok || strings.Contains(rest, "..") +} func walkGadgetFiles(t *testing.T, fn func(path string, data []byte)) { t.Helper() @@ -105,7 +115,7 @@ func TestGadgetFilesUseNeutralNames(t *testing.T) { strings.HasPrefix(trimmed, "// Copyright") || strings.HasPrefix(trimmed, "# Copyright") { continue // import paths are rewritten on a move; license headers stay } - if badInclude.MatchString(line) { + if (strings.HasSuffix(path, ".c") || strings.HasSuffix(path, ".h")) && includeOutside(line) { t.Errorf("%s:%d includes a header outside the gadget folders", path, i+1) } if m := forbiddenWords.FindString(line); m != "" { @@ -114,3 +124,18 @@ func TestGadgetFilesUseNeutralNames(t *testing.T) { } }) } + +func TestIncludeOutside(t *testing.T) { + for line, want := range map[string]bool{ + `#include "privilege_policy.h"`: false, + `#include "../common/bindings.h"`: false, + `#include `: false, + `#include "../config/x.h"`: true, + `#include "../../pkg/x.h"`: true, + `#include "../common/../../pkg/x.h"`: true, + } { + if got := includeOutside(line); got != want { + t.Errorf("%s: got %v, want %v", line, got, want) + } + } +} From 3bd49a5fe84c980a7fc401b63d8ecf2205b3ddca Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:06:58 +0200 Subject: [PATCH 09/34] test(enforce_privilege): the kernel test takes an absolute cgroup v2 path Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- .../gadgets/enforce_privilege/runtime/runtime_kernel_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go index 134d09f458..4dede884d2 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go @@ -14,7 +14,7 @@ package runtime // PRIVILEGE_IMAGE_ARCHIVE OCI archive from `ig image export` // PRIVILEGE_IMAGE image name in the archive // PRIVILEGE_CONTAINER_ID full Docker container ID -// PRIVILEGE_CGROUP_PATH its cgroup v2 path (relative to /sys/fs/cgroup) +// PRIVILEGE_CGROUP_PATH its cgroup v2 path under /sys/fs/cgroup, with a leading "/" (as in /proc//cgroup) import ( "context" From 864119c2451ec841ceb36f959f920cfd51d6a116 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:04:43 +0200 Subject: [PATCH 10/34] test(enforcement): include guard also catches spaced, absolute and nested upward includes Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- .../common/enforcement/portability_test.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/pkg/ebpf/gadgets/common/enforcement/portability_test.go b/pkg/ebpf/gadgets/common/enforcement/portability_test.go index 2a0e9a17c9..298709b2f2 100644 --- a/pkg/ebpf/gadgets/common/enforcement/portability_test.go +++ b/pkg/ebpf/gadgets/common/enforcement/portability_test.go @@ -40,15 +40,24 @@ var forbiddenWords = regexp.MustCompile(`(?i)sandbox[ _-]?polic|kubescape\.io|\b // A quoted BPF include must stay inside the gadget folders: "x.h" or // "../common/x.h" (checked by includeOutside). -var quotedInclude = regexp.MustCompile(`#include\s+"([^"]*)"`) +var quotedInclude = regexp.MustCompile(`#\s*include\s*"([^"]*)"`) func includeOutside(line string) bool { m := quotedInclude.FindStringSubmatch(line) - if m == nil || !strings.HasPrefix(m[1], "../") { + if m == nil { return false } + if strings.HasPrefix(m[1], "/") { + return true // absolute + } rest, ok := strings.CutPrefix(m[1], "../common/") - return !ok || strings.Contains(rest, "..") + if !ok { + rest = m[1] + if strings.HasPrefix(rest, "../") { + return true + } + } + return strings.Contains(rest, "..") } func walkGadgetFiles(t *testing.T, fn func(path string, data []byte)) { @@ -133,6 +142,9 @@ func TestIncludeOutside(t *testing.T) { `#include "../config/x.h"`: true, `#include "../../pkg/x.h"`: true, `#include "../common/../../pkg/x.h"`: true, + `# include "../../pkg/x.h"`: true, + `#include "/usr/include/x.h"`: true, + `#include "sub/../../x.h"`: true, } { if got := includeOutside(line); got != want { t.Errorf("%s: got %v, want %v", line, got, want) From 359b53aee312337980c978d0ed44bc15a6e5e593 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:19:44 +0200 Subject: [PATCH 11/34] fix(enforce_privilege): refuse a fork whose child cannot be armed in an enforcing slot When an armed task forks and the child's task storage mark cannot be allocated, the child escaped every rule while the instance stayed ready. Now an enforcing slot (any enforce entry) refuses the fork with -ENOMEM (a clone into another cgroup too), and counts a tracking fault per slot in arming_faults. A lost mark at exec, which cannot be refused, is counted the same way. Audit-only slots and unbound cgroups allow the fork. A new thread of an armed group leader is covered through the leader and is never refused. Moving a container to another slot arms its running tasks again. The binding controller reads the count with SlotFaults. A test-only parameter injects the failure for the new kernel test. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/common/arming.h | 60 +++++++- .../gadgets/common/enforcement/bindings.go | 60 +++++++- .../common/enforcement/bindings_test.go | 48 ++++++- .../gadgets/enforce_privilege/gadget.yaml | 4 + .../enforce_privilege/policy/kernel.go | 9 +- .../enforce_privilege/policy/kernel_test.go | 3 +- .../gadgets/enforce_privilege/program.bpf.c | 46 +++++- .../enforce_privilege/runtime/runtime.go | 1 + .../runtime/runtime_kernel_test.go | 134 ++++++++++++++---- 9 files changed, 311 insertions(+), 54 deletions(-) diff --git a/pkg/ebpf/gadgets/common/arming.h b/pkg/ebpf/gadgets/common/arming.h index d845e3d08a..029a56a216 100644 --- a/pkg/ebpf/gadgets/common/arming.h +++ b/pkg/ebpf/gadgets/common/arming.h @@ -53,8 +53,35 @@ static __always_inline bool arming_task_armed(struct task_struct *task) return a && *a; } +// Fault injection for tests only: fail every child mark at fork. Owners never +// set it in production. +const volatile bool arming_fail_fork_marks = false; + +// Tracking faults per policy slot, for the owner's health check: a mark that +// failed for a task in a bound cgroup whose slot enforces. The array is +// preallocated, so counting never fails. A new instance starts at zero and +// arms the running tasks again. +#ifndef ARMING_FAULT_SLOTS +#define ARMING_FAULT_SLOTS 64 +#endif + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, ARMING_FAULT_SLOTS); + __type(key, __u32); + __type(value, __u64); +} arming_faults SEC(".maps"); + +static __always_inline void arming_slot_fault(__u32 slot) +{ + __u64 *value = bpf_map_lookup_elem(&arming_faults, &slot); + + if (value) + __sync_fetch_and_add(value, 1); +} + // arming_mark returns false when the task storage could not be allocated (memory -// pressure): the task then stays unarmed, and the caller counts the failure. +// pressure): the task then stays unarmed, and the caller handles the failure. static __always_inline bool arming_mark(struct task_struct *task) { __u32 *a = bpf_task_storage_get(&armed_tasks, task, 0, @@ -113,12 +140,35 @@ static __always_inline bool arming_on_exec(void) return arming_mark(task); } +#define ARMING_CLONE_THREAD 0x00010000ULL + // arming_on_fork runs in task_alloc: the child inherits the parent's mark. -static __always_inline bool arming_on_fork(struct task_struct *child) +// It returns false only when the parent is armed and the child's mark failed. +// An unarmed parent (for example a runtime helper) is out of scope, not a +// failure. A new thread whose group leader is armed is covered through the +// leader (see arming_task_armed), so its own mark is best effort. +static __always_inline bool arming_on_fork(struct task_struct *child, + __u64 clone_flags) { - if (arming_task_armed(bpf_get_current_task_btf())) - return arming_mark(child); - return true; + struct task_struct *parent = bpf_get_current_task_btf(); + struct task_struct *leader; + __u32 *a; + + if (!arming_task_armed(parent)) + return true; + if (clone_flags & ARMING_CLONE_THREAD) { + leader = parent->group_leader; + a = leader ? bpf_task_storage_get(&armed_tasks, leader, 0, 0) : + NULL; + if (a && *a) { + if (!arming_fail_fork_marks) + arming_mark(child); + return true; + } + } + if (arming_fail_fork_marks) + return false; + return arming_mark(child); } #endif /* __ENFORCEMENT_ARMING_H */ diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings.go b/pkg/ebpf/gadgets/common/enforcement/bindings.go index d5bad43220..f675c3a08c 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings.go @@ -70,6 +70,18 @@ type Config struct { // ArmMap names the arming task-storage map. When set, binding a container // also arms the tasks already running in it (see ../arming.h). ArmMap string + // FaultMap names the per-slot tracking fault array (arming_faults in + // ../arming.h). When set, SlotFaults reads it. + FaultMap string +} + +// FaultMap is the part of *ebpf.Map that SlotFaults reads. +type FaultMap interface { + Lookup(key, valueOut interface{}) error + Type() ebpf.MapType + KeySize() uint32 + ValueSize() uint32 + MaxEntries() uint32 } type Status struct { @@ -90,6 +102,7 @@ type Bindings struct { desired map[execpolicy.Container]Binding active map[uint64]Binding members, armed Map + faults FaultMap ready, stopped bool startupComplete bool startupError error @@ -213,8 +226,19 @@ func (b *Bindings) bind(c Binding) error { if err := b.members.Put(id.CgroupID, bindingValue{Slot: c.Slot}); err != nil { return fmt.Errorf("writing container binding: %w", err) } - if !bound && b.armed != nil { + // A slot move arms again too: a task that the old slot left unarmed (an + // audit slot allows a fork whose child mark failed) must not escape the + // new slot. + if b.armed != nil { if err := b.armExisting(b.armed, id.CgroupPath); err != nil { + if bound && !errors.Is(err, ErrContainerGone) { + // Undo the move, so a retry moves and arms again. + if perr := b.members.Put(id.CgroupID, bindingValue{Slot: previous.Slot}); perr != nil { + b.active[id.CgroupID] = c + return errors.Join(err, perr) + } + return err + } // A container that is gone needs no protection; a startup // replay skips it (see start). if derr := b.members.Delete(id.CgroupID); derr != nil { @@ -223,6 +247,7 @@ func (b *Bindings) bind(c Binding) error { b.active[id.CgroupID] = c return errors.Join(err, derr) } + delete(b.active, id.CgroupID) return err } } @@ -245,7 +270,7 @@ func (b *Bindings) unbind(c ContainerIdentity) error { } func (b *Bindings) Start(gc operators.GadgetContext) error { - getMap := func(name string) (Map, error) { + getMap := func(name string) (*ebpf.Map, error) { value, _ := gc.GetVar(operators.MapPrefix + name) m, ok := value.(*ebpf.Map) if !ok || m == nil { @@ -261,6 +286,15 @@ func (b *Bindings) Start(gc operators.GadgetContext) error { if err != nil { return err } + var faults FaultMap + if b.config.FaultMap != "" { + if faults, err = getMap(b.config.FaultMap); err != nil { + return err + } + if faults.Type() != ebpf.Array || faults.KeySize() != 4 || faults.ValueSize() != 8 || faults.MaxEntries() < b.config.Slots { + return fmt.Errorf("invalid fault map layout") + } + } var armed Map if b.config.ArmMap != "" { if armed, err = getMap(b.config.ArmMap); err != nil { @@ -270,11 +304,27 @@ func (b *Bindings) Start(gc operators.GadgetContext) error { return fmt.Errorf("invalid arming map layout") } } - return b.start(policy, members, armed) + return b.start(policy, members, armed, faults) +} + +// SlotFaults returns the tracking faults counted for a slot since the instance +// started (see ../arming.h). A nonzero count means a task of an enforcing slot +// may be outside the rules, or was refused a fork. +func (b *Bindings) SlotFaults(slot uint32) (uint64, error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.faults == nil { + return 0, fmt.Errorf("no fault map configured") + } + var count uint64 + if err := b.faults.Lookup(slot, &count); err != nil { + return 0, fmt.Errorf("reading tracking faults: %w", err) + } + return count, nil } // start writes and freezes the policy before any binding is written. -func (b *Bindings) start(policy, members, armed Map) error { +func (b *Bindings) start(policy, members, armed Map, faults FaultMap) error { b.mu.Lock() defer b.mu.Unlock() if b.stopped { @@ -292,7 +342,7 @@ func (b *Bindings) start(policy, members, armed Map) error { if err := policy.Freeze(); err != nil { return fmt.Errorf("freezing policy: %w", err) } - b.members, b.armed = members, armed + b.members, b.armed, b.faults = members, armed, faults for _, c := range b.desired { if err := b.bind(c); err != nil && !errors.Is(err, ErrContainerGone) { return b.recordError(err) diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go index 29df707e10..43f3e8d8e6 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go @@ -67,7 +67,7 @@ func TestFreezeBeforeBindAndLifecycle(t *testing.T) { armed := &fakeMap{typ: ebpf.TaskStorage} var armedPaths []string b.armExisting = func(_ Map, path string) error { armedPaths = append(armedPaths, path); return nil } - if err := b.start(policy, members, armed); err != nil { + if err := b.start(policy, members, armed, nil); err != nil { t.Fatal(err) } if err := b.Ready(nil); err != nil { @@ -86,10 +86,17 @@ func TestFreezeBeforeBindAndLifecycle(t *testing.T) { if err := b.Bind(Binding{Container: identity}); err != nil { t.Fatal(err) } - // Moving to another slot is one map write and no second arming. - if err := b.Bind(Binding{Container: identity, Slot: 1}); err != nil || members.data[uint64(42)] != (bindingValue{Slot: 1}) || len(armedPaths) != 1 { + // Moving to another slot is one map write, and it arms again: the old + // slot may have left a task unarmed. + if err := b.Bind(Binding{Container: identity, Slot: 1}); err != nil || members.data[uint64(42)] != (bindingValue{Slot: 1}) || len(armedPaths) != 2 { t.Fatalf("slot move: %v %v armed=%v", err, members.data, armedPaths) } + // A move whose arming fails is undone, so a retry moves and arms again. + b.armExisting = func(Map, string) error { return errors.New("walk failed") } + if err := b.Bind(Binding{Container: identity, Slot: 0}); err == nil || members.data[uint64(42)] != (bindingValue{Slot: 1}) { + t.Fatalf("failed move: %v %v", err, members.data) + } + b.armExisting = func(_ Map, path string) error { armedPaths = append(armedPaths, path); return nil } if err := b.Bind(Binding{Container: identity, Slot: 2}); err == nil { t.Fatal("slot out of range must fail") } @@ -109,7 +116,7 @@ func TestFreezeBeforeBindAndLifecycle(t *testing.T) { func TestFailedArmingRemovesBinding(t *testing.T) { b, policy, members, _ := newFixture(t, func(Map) error { return nil }) b.armExisting = func(Map, string) error { return errors.New("no cgroup") } - if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}); err == nil { + if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}, nil); err == nil { t.Fatal("arming failure must fail startup") } if len(members.data) != 0 { @@ -120,11 +127,11 @@ func TestFailedArmingRemovesBinding(t *testing.T) { func TestRejectsWrongBindingLayoutAndPolicyErrors(t *testing.T) { b, policy, members, log := newFixture(t, func(Map) error { return nil }) members.max = 8 - if err := b.start(policy, members, nil); err == nil || len(*log) != 0 { + if err := b.start(policy, members, nil, nil); err == nil || len(*log) != 0 { t.Fatalf("wrong binding layout must fail before writes: %v %v", err, *log) } b, policy, members, log = newFixture(t, func(Map) error { return errors.New("bad layout") }) - if err := b.start(policy, members, nil); err == nil || len(*log) != 0 { + if err := b.start(policy, members, nil, nil); err == nil || len(*log) != 0 { t.Fatalf("policy writer error must stop startup: %v %v", err, *log) } } @@ -138,7 +145,7 @@ func TestNewValidates(t *testing.T) { func TestStartSkipsGoneContainer(t *testing.T) { b, policy, members, _ := newFixture(t, func(Map) error { return nil }) b.armExisting = func(Map, string) error { return ErrContainerGone } - if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}); err != nil { + if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}, nil); err != nil { t.Fatalf("a gone container must not fail startup: %v", err) } if err := b.Ready(nil); err != nil || len(members.data) != 0 { @@ -152,3 +159,30 @@ func TestStartSkipsGoneContainer(t *testing.T) { t.Fatalf("status %+v", st) } } + +type fakeFaults map[uint32]uint64 + +func (f fakeFaults) Lookup(k, v interface{}) error { + *v.(*uint64) = f[k.(uint32)] + return nil +} +func (fakeFaults) Type() ebpf.MapType { return ebpf.Array } +func (fakeFaults) KeySize() uint32 { return 4 } +func (fakeFaults) ValueSize() uint32 { return 8 } +func (fakeFaults) MaxEntries() uint32 { return 64 } + +func TestSlotFaults(t *testing.T) { + b, policy, members, _ := newFixture(t, func(Map) error { return nil }) + if _, err := b.SlotFaults(0); err == nil { + t.Fatal("no fault map must be an error, not zero faults") + } + if err := b.start(policy, members, nil, fakeFaults{1: 3}); err != nil { + t.Fatal(err) + } + if n, err := b.SlotFaults(0); err != nil || n != 0 { + t.Fatalf("slot 0: %d %v", n, err) + } + if n, err := b.SlotFaults(1); err != nil || n != 3 { + t.Fatalf("slot 1: %d %v", n, err) + } +} diff --git a/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml b/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml index 11eb6602bb..659bb5270b 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml +++ b/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml @@ -22,3 +22,7 @@ params: key: host-mntns-id defaultValue: "0" description: mount namespace inode of the container runtime; 0 uses the global init task's + arming_fail_fork_marks: + key: arming-fail-fork-marks + defaultValue: "false" + description: test only - fail every child arming mark at fork (fault injection) diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go index 60db0f2e7e..a292a917a4 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go @@ -309,11 +309,16 @@ func compileSlot(slot uint32, docs []Document, generation uint64, c *Compiled, o } } - entries := []KernelEntry{meta} keys := make([]KernelKey, 0, len(folded)) - for k := range folded { + for k, v := range folded { keys = append(keys, k) + // The META mode tells the program whether the slot enforces at + // all: an audit policy can carry an enforce rule. + if v.Rank == RankDenyEnforce { + meta.Value.Mode = KernelModeEnforce + } } + entries := []KernelEntry{meta} sort.Slice(keys, func(i, j int) bool { a, b := keys[i], keys[j] if a.Kind != b.Kind { diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go index d153fd10db..1511773c85 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go @@ -69,7 +69,8 @@ func TestCompileFoldsMostLimiting(t *testing.T) { t.Fatal(err) } meta, _ := lookup(c, KernelKey{Kind: KindMeta}) - if meta.Generation != 7 || meta.Mode != KernelModeAudit { + // The policy audits, but one rule enforces: the slot enforces. + if meta.Generation != 7 || meta.Mode != KernelModeEnforce { t.Fatalf("meta %+v", meta) } // Deny+enforce beats the allow on the same key, independent of order. diff --git a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c index 13d0ae97a4..c7f894dc3c 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c +++ b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c @@ -31,13 +31,16 @@ char LICENSE[] SEC("license") = "GPL"; GADGET_PARAM(host_mntns_id); +GADGET_PARAM(arming_fail_fork_marks); #define EPERM 1 +#define ENOMEM 12 #define CAP_SYS_ADMIN 21 #define CAP_OPT_NOAUDIT (1UL << 1) #define PTRACE_MODE_ATTACH 0x02 #define CLONE_NEWTIME 0x00000080ULL +#define CLONE_INTO_CGROUP 0x200000000ULL #define CLONE_NEWNS 0x00020000ULL #define CLONE_NEWCGROUP 0x02000000ULL #define CLONE_NEWUTS 0x04000000ULL @@ -461,11 +464,48 @@ static __always_inline int check_syscall(void *ctx, enum priv_hook hook, /* Arming */ +// priv_enforcing_slot returns true, with the slot, when the current task is +// in a bound cgroup whose slot enforces. +static __always_inline bool priv_enforcing_slot(__u32 *slot) +{ + struct policy_value *header; + __u64 cgid = 0; + + header = priv_header(&cgid, slot); + return header && header->mode == POLICY_MODE_ENFORCE; +} + +// The exec hook cannot fail the exec. A lost mark in an enforcing slot is a +// tracking fault: the owner reports the binding as not confirmed. SEC("lsm/bprm_committed_creds") int BPF_PROG(ig_priv_exec) { - if (!arming_on_exec()) + __u32 slot = 0; + + if (!arming_on_exec()) { verdict_count(VERDICT_COUNT_SECTION_ERROR); + if (priv_enforcing_slot(&slot)) + arming_slot_fault(slot); + } + return 0; +} + +// priv_lost_child handles an armed parent whose child could not be marked. +// An unmarked child would escape every rule, so an enforcing slot refuses the +// fork and counts a tracking fault. An audit slot or an unbound cgroup allows +// it; the failure is in the section error counter only. A clone into another +// cgroup is refused too, because the child's binding is not known here. +static __always_inline int priv_lost_child(unsigned long clone_flags) +{ + __u32 slot = 0; + + verdict_count(VERDICT_COUNT_SECTION_ERROR); + if (priv_enforcing_slot(&slot)) { + arming_slot_fault(slot); + return -ENOMEM; + } + if ((__u64)clone_flags & CLONE_INTO_CGROUP) + return -ENOMEM; return 0; } @@ -511,8 +551,8 @@ static __always_inline int do_task_alloc(void *ctx, struct task_struct *task, enum priv_syscall sc; __u64 flags; - if (!arming_on_fork(task)) - verdict_count(VERDICT_COUNT_SECTION_ERROR); + if (!arming_on_fork(task, clone_flags) && priv_lost_child(clone_flags)) + return -ENOMEM; // userns_create judges CLONE_NEWUSER. flags = clone_flags & CLONE_NS_MASK & ~CLONE_NEWUSER; diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go index df8501e046..c38d7d732e 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go @@ -126,6 +126,7 @@ func (*operator) InstantiateDataOperator(gc operators.GadgetContext, _ api.Param PolicyMap: "policy", WritePolicy: writePolicy(config.Policy.Entries), ArmMap: "armed_tasks", + FaultMap: "arming_faults", }) if err != nil { return nil, err diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go index 4dede884d2..c35adfb140 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go @@ -15,6 +15,9 @@ package runtime // PRIVILEGE_IMAGE image name in the archive // PRIVILEGE_CONTAINER_ID full Docker container ID // PRIVILEGE_CGROUP_PATH its cgroup v2 path under /sys/fs/cgroup, with a leading "/" (as in /proc//cgroup) +// +// TestForkMarkFaultInjection uses the same fixture. The container needs a +// POSIX sh. import ( "context" @@ -43,34 +46,34 @@ import ( privpolicy "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/policy" ) -func TestGadgetWithInspektorGadgetAlone(t *testing.T) { +// kernelFixture runs one gadget instance on the fixture container. +type kernelFixture struct { + ctx context.Context + id string + container ContainerIdentity + bindings *enforcement.Bindings + verdicts atomic.Int32 +} + +func newKernelFixture(t *testing.T) *kernelFixture { + t.Helper() if os.Getenv("PRIVILEGE_KERNEL_TEST") != "1" { t.Skip("explicit disposable kernel fixture opt-in required") } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - - f, err := os.Open("../examples/policy.yaml") - if err != nil { - t.Fatal(err) - } - doc, err := privpolicy.Parse(f) - f.Close() - if err != nil { - t.Fatal(err) - } - compiled, err := privpolicy.CompileKernel(doc, 1) - if err != nil { - t.Fatal(err) - } + t.Cleanup(cancel) id, cgroupPath := os.Getenv("PRIVILEGE_CONTAINER_ID"), os.Getenv("PRIVILEGE_CGROUP_PATH") var st unix.Stat_t if err := unix.Stat(filepath.Join("/sys/fs/cgroup", cgroupPath), &st); err != nil { t.Fatal(err) } - container := ContainerIdentity{Runtime: "docker", ID: id, CgroupID: st.Ino, CgroupPath: cgroupPath} + return &kernelFixture{ctx: ctx, id: id, container: ContainerIdentity{Runtime: "docker", ID: id, CgroupID: st.Ino, CgroupPath: cgroupPath}} +} - store, err := orasoci.NewFromTar(ctx, os.Getenv("PRIVILEGE_IMAGE_ARCHIVE")) +// start runs the gadget with the compiled documents and extra ebpf params. +func (f *kernelFixture) start(t *testing.T, name string, compiled privpolicy.Compiled, extra map[string]string) { + t.Helper() + store, err := orasoci.NewFromTar(f.ctx, os.Getenv("PRIVILEGE_IMAGE_ARCHIVE")) if err != nil { t.Fatal(err) } @@ -78,7 +81,7 @@ func TestGadgetWithInspektorGadgetAlone(t *testing.T) { if err := rt.Init(nil); err != nil { t.Fatal(err) } - defer rt.Close() + t.Cleanup(func() { rt.Close() }) manager, err := instancemanager.New(rt) if err != nil { t.Fatal(err) @@ -91,15 +94,13 @@ func TestGadgetWithInspektorGadgetAlone(t *testing.T) { if err := handler.Init(params); err != nil { t.Fatal(err) } - var verdicts atomic.Int32 - var bindings *enforcement.Bindings events := simple.New("privilege-test-verdicts", simple.WithPriority(100), simple.OnInit(func(gc operators.GadgetContext) error { var err error - if bindings, err = GetBindings(gc); err != nil { + if f.bindings, err = GetBindings(gc); err != nil { return err } return gc.GetDataSources()["verdict"].Subscribe(func(datasource.DataSource, datasource.Data) error { - verdicts.Add(1) + f.verdicts.Add(1) return nil }, 100) })) @@ -107,33 +108,104 @@ func TestGadgetWithInspektorGadgetAlone(t *testing.T) { if err != nil { t.Fatal(err) } - desired := &api.GadgetInstance{Id: "privilege-kernel-test", GadgetConfig: &api.GadgetRunRequest{Version: api.VersionGadgetRunProtocol, ImageName: os.Getenv("PRIVILEGE_IMAGE"), - ParamValues: map[string]string{"operator.oci.ebpf.host-mntns-id": strconv.FormatUint(runtimeMntns, 10)}}} - if err := manager.StartGadget(ctx, desired, gadgetcontext.WithOrasReadonlyTarget(store), + values := map[string]string{"operator.oci.ebpf.host-mntns-id": strconv.FormatUint(runtimeMntns, 10)} + for k, v := range extra { + values["operator.oci.ebpf."+k] = v + } + desired := &api.GadgetInstance{Id: name, GadgetConfig: &api.GadgetRunRequest{Version: api.VersionGadgetRunProtocol, ImageName: os.Getenv("PRIVILEGE_IMAGE"), ParamValues: values}} + if err := manager.StartGadget(f.ctx, desired, gadgetcontext.WithOrasReadonlyTarget(store), gadgetcontext.WithAdditionalDataOperators(handler, New(), events), WithConfig(Config{Policy: compiled})); err != nil { t.Fatal(err) } - defer manager.StopGadget(context.Background(), desired.Id) + t.Cleanup(func() { manager.StopGadget(context.Background(), desired.Id) }) +} - unshare := func() error { return exec.CommandContext(ctx, "docker", "exec", id, "unshare", "-m", "true").Run() } +func (f *kernelFixture) exec(args ...string) error { + return exec.CommandContext(f.ctx, "docker", append([]string{"exec", f.id}, args...)...).Run() +} + +func TestGadgetWithInspektorGadgetAlone(t *testing.T) { + f := newKernelFixture(t) + file, err := os.Open("../examples/policy.yaml") + if err != nil { + t.Fatal(err) + } + doc, err := privpolicy.Parse(file) + file.Close() + if err != nil { + t.Fatal(err) + } + compiled, err := privpolicy.CompileKernel(doc, 1) + if err != nil { + t.Fatal(err) + } + f.start(t, "privilege-kernel-test", compiled, nil) + + unshare := func() error { return f.exec("unshare", "-m", "true") } if err := unshare(); err != nil { t.Fatalf("unbound container must not be judged: %v", err) } - if err := bindings.Bind(enforcement.Binding{Container: container}); err != nil { + if err := f.bindings.Bind(enforcement.Binding{Container: f.container}); err != nil { t.Fatal(err) } if unshare() == nil { t.Fatal("the example policy denies unshare(CLONE_NEWNS) in a bound container") } - if err := bindings.Unbind(container); err != nil { + if err := f.bindings.Unbind(f.container); err != nil { t.Fatal(err) } if err := unshare(); err != nil { t.Fatalf("unbound again must be allowed: %v", err) } time.Sleep(500 * time.Millisecond) - if verdicts.Load() == 0 { + if f.verdicts.Load() == 0 { t.Fatal("no verdict event for the deny") } } + +// TestForkMarkFaultInjection makes every child arming mark fail at fork. An +// armed shell that forks must then be refused in enforce mode, with a tracking +// fault for the slot, and allowed in audit mode, with no tracking fault. +func TestForkMarkFaultInjection(t *testing.T) { + for _, mode := range []string{"enforce", "audit"} { + t.Run(mode, func(t *testing.T) { + f := newKernelFixture(t) + doc := privpolicy.Document{Version: 1, Name: "fault-" + mode, Policy: privpolicy.Policy{Mode: mode, Privilege: privpolicy.Section{Default: "allow"}}} + compiled, err := privpolicy.CompileKernel(doc, 1) + if err != nil { + t.Fatal(err) + } + f.start(t, "privilege-fault-"+mode, compiled, map[string]string{"arming-fail-fork-marks": "true"}) + fork := func() error { return f.exec("sh", "-c", "true & wait") } + if err := fork(); err != nil { + t.Fatalf("an unbound container must not be judged: %v", err) + } + if err := f.bindings.Bind(enforcement.Binding{Container: f.container}); err != nil { + t.Fatal(err) + } + defer f.bindings.Unbind(f.container) + err = fork() + faults, ferr := f.bindings.SlotFaults(0) + if ferr != nil { + t.Fatal(ferr) + } + switch mode { + case "enforce": + if err == nil { + t.Fatal("a fork whose child cannot be armed must be refused") + } + if faults == 0 { + t.Fatal("the refused fork must count a tracking fault") + } + case "audit": + if err != nil { + t.Fatalf("audit mode must allow the fork: %v", err) + } + if faults != 0 { + t.Fatalf("audit faults must not block: %d", faults) + } + } + }) + } +} From 0c07186880491c3eda599a325d6bbecc915559f5 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:19:44 +0200 Subject: [PATCH 12/34] test(merge): shared merge contract and test vectors One section-neutral vector file (policy combination to expected verdict) and a CONTRACT doc that states the merge rule and where it differs from the CRD text. The privilege compiler tests run the vectors. No change to the semantics. The README documents the lost-arming handling, the child cgroup scope and the follow-ups. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/common/README.md | 5 +- pkg/ebpf/gadgets/common/merge/CONTRACT.md | 35 +++++ pkg/ebpf/gadgets/common/merge/vectors.yaml | 107 +++++++++++++++ pkg/ebpf/gadgets/enforce_privilege/README.md | 42 ++++-- .../enforce_privilege/policy/contract_test.go | 126 ++++++++++++++++++ 5 files changed, 305 insertions(+), 10 deletions(-) create mode 100644 pkg/ebpf/gadgets/common/merge/CONTRACT.md create mode 100644 pkg/ebpf/gadgets/common/merge/vectors.yaml create mode 100644 pkg/ebpf/gadgets/enforce_privilege/policy/contract_test.go diff --git a/pkg/ebpf/gadgets/common/README.md b/pkg/ebpf/gadgets/common/README.md index 0359e1792f..cce63570ee 100644 --- a/pkg/ebpf/gadgets/common/README.md +++ b/pkg/ebpf/gadgets/common/README.md @@ -17,8 +17,9 @@ and keeps its path rules in an array that `policy.h` does not describe. | `bindings.h` | `bindings` map: cgroup id → the container's policy slot; optional parent-cgroup depth | | `policy.h` | policy modes (0 off, 1 audit, 2 enforce), rank resolution (explicit deny > allow > default, one mode per rule), shared policy value | | `verdict.h` | first verdict fields (`generation`, `cgroup`, `deny`, `mode`), per-CPU `counters` (denied, submitted, dropped, allowed, section error, rate limited), 100 reports per CPU per second | -| `arming.h` | arming rule: judge only tasks that executed outside the runtime mount namespace, and their children | -| `enforcement/` | Go: the binding controller of one node-wide instance (policy written and frozen before any binding, live `Bind`/`Unbind` of a container to a slot, readiness, arming of tasks already running) | +| `arming.h` | arming rule: judge only tasks that executed outside the runtime mount namespace, and their children; `arming_faults`, tracking faults per slot | +| `merge/` | the merge contract for several policies on one container, and its shared test vectors | +| `enforcement/` | Go: the binding controller of one node-wide instance (policy written and frozen before any binding, live `Bind`/`Unbind` of a container to a slot, readiness, arming of tasks already running, `SlotFaults`) | Section-specific parts (the policy key layout, hooks, compiler) stay in each gadget. diff --git a/pkg/ebpf/gadgets/common/merge/CONTRACT.md b/pkg/ebpf/gadgets/common/merge/CONTRACT.md new file mode 100644 index 0000000000..0043693e17 --- /dev/null +++ b/pkg/ebpf/gadgets/common/merge/CONTRACT.md @@ -0,0 +1,35 @@ +# Merge contract + +Several policies can apply to one container. Every section compiler merges them +with the same rule, and `vectors.yaml` holds the shared test vectors. A section +compiler that merges policies runs the vectors in its tests (the privilege +compiler: `enforce_privilege/policy/contract_test.go`). + +## Rule + +1. Rules are the union of every policy's rules. Each rule keeps its own mode + (else its policy's mode). +2. On one key the most limiting rule wins: deny (enforce) > deny (audit) > + allow. +3. When no rule matches a key, the section default applies. The merged default + is deny if any policy's default is deny, else allow. Its mode is the + strictest mode of the policies whose default equals the merged default. +4. An explicit allow beats another policy's default deny. +5. An audit deny never weakens another policy's enforce default deny. When a + key is denied by rules only in audit mode, the enforce default deny of each + policy without a rule on that key still applies. So adding an audit policy + never makes the merged result less strict. +6. Qualified keys (privilege: namespace flags) are judged per qualifier, with + rules 1–5 for each qualifier. + +## Where it differs from the CRD text + +The CRD design says the section defaults apply only when no rule matches. Read +literally, a matching audit deny of one policy then hides another policy's +enforce default deny, and the merged result is audit only. Rule 5 differs: it +keeps the enforce default deny. The vector "an audit deny never weakens another +policy's enforce default deny" covers it. + +Rule 5 is pending a product decision. Until then, a change to it changes +`vectors.yaml` and this file in the same commit, so the sections cannot diverge +silently. diff --git a/pkg/ebpf/gadgets/common/merge/vectors.yaml b/pkg/ebpf/gadgets/common/merge/vectors.yaml new file mode 100644 index 0000000000..37b1421002 --- /dev/null +++ b/pkg/ebpf/gadgets/common/merge/vectors.yaml @@ -0,0 +1,107 @@ +# Merge contract vectors (see CONTRACT.md): a combination of policies on one +# container and the verdict expected for each call. Every section compiler that +# merges policies runs these vectors in its tests. +# +# Section-neutral names: +# key a rule target (privilege: a syscall; k1 = mount, k2 = unshare) +# qual a qualifier that narrows a key (privilege: a namespace flag; +# q1 = CLONE_NEWNS, q2 = CLONE_NEWNET). A rule without qual +# matches every qualifier. A call with quals is judged per +# qualifier and the most limiting result wins. +# verdict allow | deny-audit | deny-enforce +# A rule without mode takes its policy's mode. +vectors: + - name: one policy, enforce default deny + policies: + - {mode: enforce, default: deny} + checks: + - {key: k1, verdict: deny-enforce} + + - name: one policy, its own audit rule beats its default + policies: + - mode: enforce + default: deny + rules: + - {key: k1, action: deny, mode: audit} + checks: + - {key: k1, verdict: deny-audit} + + - name: an audit deny never weakens another policy's enforce default deny + policies: + - mode: audit + default: allow + rules: + - {key: k1, action: deny} + - {mode: enforce, default: deny} + checks: + - {key: k1, verdict: deny-enforce} + + - name: two audit policies stay audit + policies: + - mode: audit + default: allow + rules: + - {key: k1, action: deny} + - {mode: audit, default: deny} + checks: + - {key: k1, verdict: deny-audit} + + - name: an explicit allow beats another policy's default deny + policies: + - mode: enforce + default: allow + rules: + - {key: k1, action: allow} + - {mode: enforce, default: deny} + checks: + - {key: k1, verdict: allow} + - {key: k2, quals: [q1], verdict: deny-enforce} + + - name: an explicit deny beats another policy's allow + policies: + - mode: enforce + default: allow + rules: + - {key: k1, action: deny} + - mode: enforce + default: allow + rules: + - {key: k1, action: allow} + checks: + - {key: k1, verdict: deny-enforce} + + - name: the merged default takes the mode of the policies that agree + policies: + - {mode: audit, default: deny} + - {mode: enforce, default: allow} + checks: + - {key: k1, verdict: deny-audit} + + - name: per qualifier, an audit deny on one qualifier under an allow of the key + policies: + - mode: audit + default: allow + rules: + - {key: k2, qual: q1, action: deny} + - mode: enforce + default: deny + rules: + - {key: k2, action: allow} + checks: + - {key: k2, quals: [q1], verdict: deny-audit} + - {key: k2, quals: [q2], verdict: allow} + + - name: per qualifier, an audit deny of the key under an allow of one qualifier + policies: + - mode: audit + default: allow + rules: + - {key: k2, action: deny} + - mode: enforce + default: deny + rules: + - {key: k2, qual: q1, action: allow} + checks: + - {key: k2, quals: [q1], verdict: deny-audit} + - {key: k2, quals: [q2], verdict: deny-enforce} + - {key: k2, quals: [q1, q2], verdict: deny-enforce} diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index 920bdb2658..5f4dcd479c 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -16,7 +16,9 @@ node: - The policy map is frozen per instance. A combination the running instance does not have (a new or edited policy) needs a new instance: the owner starts one with every slot in use and every container bound, and then stops the old - one, so both enforce during the overlap. + one, so both enforce during the overlap. This whole-instance replacement is a + staging choice: the owner hides it behind its backend interface, so a later + in-place policy update can replace it. - An owner can create a slot before a container starts, so the container binds with a live write. @@ -73,7 +75,9 @@ policy: own name, origin and mode); a default is deny if any document denies, with the strictest mode among the documents that agree; a monitoring (audit) deny rule never weakens another document's enforce deny, including its enforce default - deny; an explicit allow still beats another document's default deny. + deny; an explicit allow still beats another document's default deny. The + rule and its test vectors are shared with other sections: + [merge contract](../common/merge/CONTRACT.md). ## How it decides @@ -86,7 +90,23 @@ policy: `host-mntns-id` gadget parameter; the owner sets it to the mount namespace of PID 1 as it sees it (in kind, the node container's). When it is 0, the program uses the global init task's. -- **Scope.** A bound cgroup also covers up to 31 levels of child cgroups. +- **Lost arming.** The kernel can fail to allocate a task's mark (memory + pressure). When an armed task forks and the child's mark fails, an enforcing + slot (one with any enforce entry) refuses the fork (`-ENOMEM`): an unmarked + child would escape every rule. A clone into another cgroup + (`CLONE_INTO_CGROUP`) is refused the same way. An audit-only slot, or an + unbound cgroup, allows the fork. A new thread of an armed group leader is + covered through the leader, so it is never refused. An unarmed parent (for + example a runtime helper) is out of scope, not a failure. Every failed mark is + counted in `counters` slot 4. Moving a container to another slot arms its + running tasks again. +- **Tracking faults.** A failed mark in an enforcing slot (a refused fork, or a + lost mark at exec, which cannot be refused) is also counted per slot in + `arming_faults`. The owner reads it with `Bindings.SlotFaults`: a nonzero + count means the slot's protection is not confirmed. A new instance starts at + zero and arms the running tasks again. Audit-only faults do not count there. +- **Scope.** A bound cgroup also covers up to 31 levels of child cgroups. This is + by design and differs from enforce_exec, which binds the exact cgroup only. - **Capabilities.** A `capable()` call is judged only when the task uses its own credentials (overlayfs copy-up uses the mounter's) and the call is not a no-audit probe. @@ -99,10 +119,16 @@ policy: ## Known limits -- **Failed arming mark.** If the kernel cannot allocate task storage (memory - pressure), the task stays unarmed and is not judged until its next exec. The - failure is counted in `counters` slot 4. A later check cannot tell such a - task apart from the runtime's own setup, so it is not retried. +- **Lost mark at exec.** The exec hook cannot fail the exec, so a task whose + mark fails there stays unarmed until its next exec. In an enforcing slot this + is a tracking fault (see above). A later check cannot tell such a task apart + from the runtime's own setup, so the gadget does not retry it; a new instance + arms it again. +- **Fault injection.** The `arming-fail-fork-marks` parameter makes every child + mark fail at fork. It exists for tests (`runtime/runtime_kernel_test.go`); + owners do not set it. +- **Not in this gadget yet:** export of verdicts as events (the owner logs + them), and kernels older than 6.1. - **No-audit capability checks** (`CAP_OPT_NOAUDIT`) are excluded from capability enforcement. Most are kernel probes (for example the overcommit check), and judging them would deny normal work in every bound container. Some @@ -127,7 +153,7 @@ policy: | Map | Content | |---|---| | `policy` | frozen hash: `{slot, kind, id, qual, name}` → `{rank, rule_id, origin_id}`; each slot's META entry holds generation and mode (`privilege_policy.h`) | -| `bindings`, `armed_tasks`, `counters` | from the common library | +| `bindings`, `armed_tasks`, `arming_faults`, `counters` | from the common library | The `verdict` datasource shares `generation`, `cgroup`, `deny`, `mode` and `host_pid` with enforce_exec and adds `rule_id`, `origin_id`, `is_default`, diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/contract_test.go b/pkg/ebpf/gadgets/enforce_privilege/policy/contract_test.go new file mode 100644 index 0000000000..7a4a52b67e --- /dev/null +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/contract_test.go @@ -0,0 +1,126 @@ +// Copyright 2026 The Kubescape Authors +// SPDX-License-Identifier: Apache-2.0 + +package privpolicy + +import ( + "bytes" + "os" + "testing" + + "gopkg.in/yaml.v3" +) + +// The shared merge contract vectors (../../common/merge/CONTRACT.md), mapped to +// privilege targets. +var ( + contractKeys = map[string]string{"k1": "mount", "k2": "unshare"} + contractQuals = map[string]string{"q1": "CLONE_NEWNS", "q2": "CLONE_NEWNET"} + contractRanks = map[string]uint8{"allow": RankAllow, "deny-audit": RankDenyAudit, "deny-enforce": RankDenyEnforce} +) + +type contractVectors struct { + Vectors []struct { + Name string `yaml:"name"` + Policies []struct { + Mode string `yaml:"mode"` + Default string `yaml:"default"` + Rules []struct { + Key string `yaml:"key"` + Qual string `yaml:"qual"` + Action string `yaml:"action"` + Mode string `yaml:"mode"` + } `yaml:"rules"` + } `yaml:"policies"` + Checks []struct { + Key string `yaml:"key"` + Quals []string `yaml:"quals"` + Verdict string `yaml:"verdict"` + } `yaml:"checks"` + } `yaml:"vectors"` +} + +// contractRank resolves a call like the program: per qualifier, the rules on +// the bare key and on the qualified key, else the default; the most limiting +// qualifier wins. +func contractRank(c Compiled, syscall string, flags []string) uint8 { + def, _ := lookup(c, KernelKey{Kind: KindDefault, ID: KindSyscall}) + bare, _ := lookup(c, KernelKey{Kind: KindSyscall, ID: Syscalls[syscall]}) + if len(flags) == 0 { + if bare.Rank != RankNone { + return bare.Rank + } + return def.Rank + } + var result uint8 + for _, f := range flags { + q, _ := lookup(c, KernelKey{Kind: KindSyscall, ID: Syscalls[syscall], Qual: NamespaceFlags[f]}) + r := max(bare.Rank, q.Rank) + if r == RankNone { + r = def.Rank + } + result = max(result, r) + } + return result +} + +func TestMergeContractVectors(t *testing.T) { + data, err := os.ReadFile("../../common/merge/vectors.yaml") + if err != nil { + t.Fatal(err) + } + var file contractVectors + decoder := yaml.NewDecoder(bytes.NewReader(data)) + decoder.KnownFields(true) + if err := decoder.Decode(&file); err != nil { + t.Fatal(err) + } + key := func(k string) string { + if contractKeys[k] == "" { + t.Fatalf("unknown key %q", k) + } + return contractKeys[k] + } + qual := func(q string) string { + if contractQuals[q] == "" { + t.Fatalf("unknown qual %q", q) + } + return contractQuals[q] + } + if len(file.Vectors) == 0 { + t.Fatal("no vectors") + } + for _, v := range file.Vectors { + t.Run(v.Name, func(t *testing.T) { + var docs []Document + for i, p := range v.Policies { + d := Document{Version: 1, Name: string(rune('a' + i)), Policy: Policy{Mode: p.Mode, Privilege: Section{Default: p.Default}}} + for j, r := range p.Rules { + rule := Rule{Name: "r" + string(rune('0'+j)), Action: r.Action, Mode: r.Mode, Target: Target{Syscall: []string{key(r.Key)}}} + if r.Qual != "" { + rule.Target.Flags = []string{qual(r.Qual)} + } + d.Policy.Privilege.Rules = append(d.Policy.Privilege.Rules, rule) + } + docs = append(docs, d) + } + c, err := CompileSlots([][]Document{docs}, 1) + if err != nil { + t.Fatal(err) + } + for _, check := range v.Checks { + var flags []string + for _, q := range check.Quals { + flags = append(flags, qual(q)) + } + want, ok := contractRanks[check.Verdict] + if !ok { + t.Fatalf("unknown verdict %q", check.Verdict) + } + if got := contractRank(c, key(check.Key), flags); got != want { + t.Errorf("%s %v: rank %d, want %s", check.Key, check.Quals, got, check.Verdict) + } + } + }) + } +} From 38743dd3778951103510b81127f428efef288b79 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:28:16 +0200 Subject: [PATCH 13/34] fix(enforcement): a same-slot retry arms again after a failed arming When arming failed and the kernel entry could not be removed (or a slot move could not be undone), the binding stayed tracked, and a retry with the same slot returned early without arming. The controller now marks such cgroups as unarmed, so the retry arms them. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- .../gadgets/common/enforcement/bindings.go | 20 +++++++++---- .../common/enforcement/bindings_test.go | 30 +++++++++++++++++++ 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings.go b/pkg/ebpf/gadgets/common/enforcement/bindings.go index f675c3a08c..e578bcaf68 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings.go @@ -97,10 +97,13 @@ type Status struct { // Bindings serializes all writes. Map handles are borrowed from the eBPF operator. type Bindings struct { - mu sync.Mutex - config Config - desired map[execpolicy.Container]Binding - active map[uint64]Binding + mu sync.Mutex + config Config + desired map[execpolicy.Container]Binding + active map[uint64]Binding + // unarmed marks bound cgroups whose running tasks the last bind did + // not arm, so a retry with the same slot arms them. + unarmed map[uint64]bool members, armed Map faults FaultMap ready, stopped bool @@ -118,7 +121,7 @@ func New(config Config) (*Bindings, error) { if len(config.InitialBindings) > MaxBindings { return nil, fmt.Errorf("too many initial bindings") } - b := &Bindings{config: config, desired: map[execpolicy.Container]Binding{}, active: map[uint64]Binding{}, armExisting: ArmExisting} + b := &Bindings{config: config, desired: map[execpolicy.Container]Binding{}, active: map[uint64]Binding{}, unarmed: map[uint64]bool{}, armExisting: ArmExisting} for _, c := range config.InitialBindings { if err := b.validate(c); err != nil { return nil, err @@ -210,7 +213,7 @@ func (b *Bindings) bind(c Binding) error { if bound && previous.Container != id { return fmt.Errorf("cgroup identity conflicts with active binding") } - if bound && previous.Slot == c.Slot { + if bound && previous.Slot == c.Slot && !b.unarmed[id.CgroupID] { return nil } if !bound { @@ -235,6 +238,7 @@ func (b *Bindings) bind(c Binding) error { // Undo the move, so a retry moves and arms again. if perr := b.members.Put(id.CgroupID, bindingValue{Slot: previous.Slot}); perr != nil { b.active[id.CgroupID] = c + b.unarmed[id.CgroupID] = true return errors.Join(err, perr) } return err @@ -245,12 +249,15 @@ func (b *Bindings) bind(c Binding) error { // The kernel entry stays: keep it tracked so Unbind can // remove it later. b.active[id.CgroupID] = c + b.unarmed[id.CgroupID] = true return errors.Join(err, derr) } delete(b.active, id.CgroupID) + delete(b.unarmed, id.CgroupID) return err } } + delete(b.unarmed, id.CgroupID) b.active[id.CgroupID] = c return nil } @@ -266,6 +273,7 @@ func (b *Bindings) unbind(c ContainerIdentity) error { return fmt.Errorf("removing container binding: %w", err) } delete(b.active, c.CgroupID) + delete(b.unarmed, c.CgroupID) return nil } diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go index 43f3e8d8e6..d9b20c6384 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/cilium/ebpf" + execpolicy "github.com/inspektor-gadget/inspektor-gadget/gadgets/enforce_exec/policy" "golang.org/x/sys/unix" ) @@ -19,6 +20,7 @@ type fakeMap struct { frozen bool log *[]string name string + deleteErr error } func (m *fakeMap) Put(k, v interface{}) error { @@ -31,6 +33,9 @@ func (m *fakeMap) Put(k, v interface{}) error { } func (m *fakeMap) Delete(k interface{}) error { + if m.deleteErr != nil { + return m.deleteErr + } if _, ok := m.data[k]; !ok { return ebpf.ErrKeyNotExist } @@ -124,6 +129,31 @@ func TestFailedArmingRemovesBinding(t *testing.T) { } } +// When arming fails and the kernel entry cannot be removed, the binding stays +// tracked. A retry with the same slot must arm again, not return early. +func TestSameSlotRetryArmsAfterFailedArming(t *testing.T) { + b, policy, members, _ := newFixture(t, func(Map) error { return nil }) + b.config.InitialBindings, b.desired = nil, map[execpolicy.Container]Binding{} + arms := 0 + b.armExisting = func(Map, string) error { arms++; return nil } + if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}, nil); err != nil { + t.Fatal(err) + } + b.armExisting = func(Map, string) error { arms++; return errors.New("walk failed") } + members.deleteErr = errors.New("busy") + if err := b.Bind(Binding{Container: identity}); err == nil { + t.Fatal("a failed arming must fail the bind") + } + b.armExisting = func(Map, string) error { arms++; return nil } + members.deleteErr = nil + if err := b.Bind(Binding{Container: identity}); err != nil || arms != 2 { + t.Fatalf("retry must arm: %v arms=%d", err, arms) + } + if err := b.Bind(Binding{Container: identity}); err != nil || arms != 2 { + t.Fatalf("an armed binding needs no new arming: %v arms=%d", err, arms) + } +} + func TestRejectsWrongBindingLayoutAndPolicyErrors(t *testing.T) { b, policy, members, log := newFixture(t, func(Map) error { return nil }) members.max = 8 From 0a469cf01666f31b6606b6c152d2e40e119b3fe0 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:28:16 +0200 Subject: [PATCH 14/34] fix(enforce_privilege): reject clone and clone3 rules without flags The gadget judges a clone by its namespace flags, so a plain fork is never judged. A clone or clone3 rule without flags read as "deny every fork" but did not block one. Blocking every fork is not a useful policy, so the compiler now rejects such a rule with a clear error. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/enforce_privilege/README.md | 3 ++- .../gadgets/enforce_privilege/policy/kernel_test.go | 2 ++ pkg/ebpf/gadgets/enforce_privilege/policy/policy.go | 10 ++++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index 5f4dcd479c..a20dd5ebd4 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -57,7 +57,8 @@ policy: ``` - A rule targets exactly one kind: `syscall` (optionally with `flags` for - `unshare`, `setns`, `clone`, `clone3`), `capability` (short names such as + `unshare`, `setns`, `clone`, `clone3`; required for `clone` and `clone3`, + because a plain fork is never judged), `capability` (short names such as `SYS_ADMIN`) or `mountFsType`. - Resolution: an explicit deny beats an allow, an allow beats the default, independent of rule order. Each rule has its own mode. Each namespace flag of diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go index 1511773c85..c9fc7da67b 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go @@ -125,6 +125,8 @@ func TestValidationErrors(t *testing.T) { "NUL fstype": " - {name: r, target: {mountFsType: [\"pro\\0c\"]}, action: deny}\n", "duplicate name": " - {name: r, target: {syscall: [mount]}, action: deny}\n - {name: r, target: {syscall: [bpf]}, action: deny}\n", "unknown field": " - {name: r, target: {path: [/x]}, action: deny}\n", + "bare clone": " - {name: r, target: {syscall: [clone]}, action: deny}\n", + "bare clone3": " - {name: r, target: {syscall: [unshare, clone3]}, action: allow}\n", } { if _, err := parse(base + rule); err == nil { t.Errorf("%s: expected error", name) diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go b/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go index d44d2dd39c..92b018dd3a 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go @@ -133,6 +133,16 @@ func (t Target) validate() error { return fmt.Errorf("unsupported syscall %q", s) } } + if len(t.Flags) == 0 { + for _, s := range t.Syscall { + if s == "clone" || s == "clone3" { + // The gadget judges a clone by its namespace flags. A rule + // without flags would read as "deny every fork", which it + // does not do. + return fmt.Errorf("a %s rule must list namespace flags: a plain fork is never judged", s) + } + } + } if len(t.Flags) > 0 { if len(t.Syscall) == 0 { return fmt.Errorf("flags qualify a syscall target only") From b7e633608674ff809a3319620d361be7a65f7bdc Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:37:35 +0200 Subject: [PATCH 15/34] test(enforcement): a slot move retry arms again after a failed rollback Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- .../common/enforcement/bindings_test.go | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go index d9b20c6384..8cf052426e 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go @@ -20,10 +20,13 @@ type fakeMap struct { frozen bool log *[]string name string - deleteErr error + deleteErr, putErr error } func (m *fakeMap) Put(k, v interface{}) error { + if m.putErr != nil { + return m.putErr + } if m.frozen { return unix.EPERM } @@ -154,6 +157,33 @@ func TestSameSlotRetryArmsAfterFailedArming(t *testing.T) { } } +// A slot move whose arming fails and whose rollback also fails keeps the new +// slot tracked. A later retry with that slot must arm again. +func TestSlotMoveRetryArmsAfterFailedRollback(t *testing.T) { + b, policy, members, _ := newFixture(t, func(Map) error { return nil }) + arms := 0 + b.armExisting = func(Map, string) error { arms++; return nil } + if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}, nil); err != nil { + t.Fatal(err) + } + b.armExisting = func(Map, string) error { + arms++ + members.putErr = errors.New("busy") // the rollback write fails + return errors.New("walk failed") + } + if err := b.Bind(Binding{Container: identity, Slot: 1}); err == nil { + t.Fatal("a failed move must fail the bind") + } + members.putErr = nil + b.armExisting = func(Map, string) error { arms++; return nil } + if err := b.Bind(Binding{Container: identity, Slot: 1}); err != nil || arms != 3 { + t.Fatalf("retry must arm: %v arms=%d", err, arms) + } + if members.data[uint64(42)] != (bindingValue{Slot: 1}) { + t.Fatalf("binding %v", members.data) + } +} + func TestRejectsWrongBindingLayoutAndPolicyErrors(t *testing.T) { b, policy, members, log := newFixture(t, func(Map) error { return nil }) members.max = 8 From b307ca1671c679e216ca757d1d573fb2da4e7522 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:09:23 +0200 Subject: [PATCH 16/34] fix(enforcement): arm the group leader of a thread that cannot be opened On kernels without PIDFD_THREAD a non-leader thread has no pidfd, and the BPF rule judges it through its group leader. The walk now arms that leader, which can be outside the walked cgroup. A leader in the runtime mount namespace is not armed; the walk then returns an error. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- .../gadgets/common/enforcement/arm_linux.go | 43 +++++++++++++++++-- .../common/enforcement/arm_linux_test.go | 42 ++++++++++++++++++ 2 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 pkg/ebpf/gadgets/common/enforcement/arm_linux_test.go diff --git a/pkg/ebpf/gadgets/common/enforcement/arm_linux.go b/pkg/ebpf/gadgets/common/enforcement/arm_linux.go index 699fb6d4ed..bf7ac52d57 100644 --- a/pkg/ebpf/gadgets/common/enforcement/arm_linux.go +++ b/pkg/ebpf/gadgets/common/enforcement/arm_linux.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "golang.org/x/sys/unix" @@ -105,10 +106,14 @@ func armThreads(armed Map, threads string, hostMntns uint64, seen map[int]bool) if errors.Is(err, unix.EINVAL) { fd, err = unix.PidfdOpen(tid, 0) } - if errors.Is(err, unix.ESRCH) || errors.Is(err, unix.EINVAL) { - // Exited, or a non-leader thread on a kernel without - // PIDFD_THREAD: the BPF rule judges it through its leader. - seen[tid] = true + if errors.Is(err, unix.EINVAL) { + // A non-leader thread on a kernel without PIDFD_THREAD: the + // BPF rule judges it through its group leader, which can be + // outside this cgroup, so arm the leader. + fd, err = openLeader(scanner.Text(), hostMntns) + } + if errors.Is(err, unix.ESRCH) || errors.Is(err, fs.ErrNotExist) { + seen[tid] = true // exited continue } if err != nil { @@ -124,6 +129,36 @@ func armThreads(armed Map, threads string, hostMntns uint64, seen map[int]bool) return scanner.Err() } +// openLeader opens a pidfd of the thread group leader of tid. A leader in the +// runtime mount namespace is a runtime helper: arming it would judge the +// runtime, so the thread cannot be armed and the caller gets an error. +func openLeader(tid string, hostMntns uint64) (int, error) { + data, err := os.ReadFile(filepath.Join(host.HostProcFs, tid, "status")) + if err != nil { + return -1, err + } + tgid := -1 + for _, line := range strings.Split(string(data), "\n") { + if v, ok := strings.CutPrefix(line, "Tgid:"); ok { + tgid, err = strconv.Atoi(strings.TrimSpace(v)) + if err != nil { + return -1, fmt.Errorf("reading leader of task %s: %w", tid, err) + } + } + } + if tgid <= 0 { + return -1, fmt.Errorf("no leader for task %s", tid) + } + mntns, err := mntnsOf(strconv.Itoa(tgid)) + if err != nil { + return -1, err + } + if mntns == hostMntns { + return -1, fmt.Errorf("thread %s has a runtime helper as leader", tid) + } + return unix.PidfdOpen(tgid, 0) +} + // RuntimeMountNamespace is the mount namespace of PID 1 as this process sees // it: the namespace the container runtime runs in. Pass it to the gadget's // host_mntns_id so the BPF arming rule and ArmExisting agree. diff --git a/pkg/ebpf/gadgets/common/enforcement/arm_linux_test.go b/pkg/ebpf/gadgets/common/enforcement/arm_linux_test.go new file mode 100644 index 0000000000..8e5b78c27f --- /dev/null +++ b/pkg/ebpf/gadgets/common/enforcement/arm_linux_test.go @@ -0,0 +1,42 @@ +// Copyright 2026 The Kubescape Authors +// SPDX-License-Identifier: Apache-2.0 + +package enforcement + +import ( + "os" + "strconv" + "testing" + + "golang.org/x/sys/unix" +) + +func TestOpenLeaderOfThread(t *testing.T) { + entries, err := os.ReadDir("/proc/self/task") + if err != nil { + t.Skip("no /proc") + } + pid := strconv.Itoa(os.Getpid()) + thread := "" + for _, e := range entries { + if e.Name() != pid { + thread = e.Name() + } + } + if thread == "" { + t.Skip("no second thread") + } + fd, err := openLeader(thread, 0) + if err != nil { + t.Fatalf("leader of thread %s: %v", thread, err) + } + unix.Close(fd) + own, err := mntnsOf(pid) + if err != nil { + t.Fatal(err) + } + if fd, err := openLeader(thread, own); err == nil { + unix.Close(fd) + t.Fatal("a leader in the runtime mount namespace must not be armed") + } +} From f6d9b1d9cd8ba4fa0971bc0d3fc110d74c83a244 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:09:23 +0200 Subject: [PATCH 17/34] fix(enforce_privilege): a slot enforces only when one of its entries enforces The META mode was enforce for any document in enforce mode, so a slot with only allow entries refused a fork whose child mark failed. It is now enforce only when an entry has the enforce deny rank, as the README says. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- .../gadgets/enforce_privilege/policy/kernel.go | 14 +++++--------- .../enforce_privilege/policy/kernel_test.go | 7 ++++++- .../runtime/runtime_kernel_test.go | 4 +++- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go index a292a917a4..2cda0ae362 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go @@ -178,15 +178,10 @@ func compileSlot(slot uint32, docs []Document, generation uint64, c *Compiled, o if err := d.Validate(); err != nil { return nil, err } - switch d.Policy.Mode { - case "enforce": - meta.Value.Mode = KernelModeEnforce - case "audit": - if meta.Value.Mode == KernelModeOff { - meta.Value.Mode = KernelModeAudit - } - } + // The META mode is audit for any active document and enforce only + // when an entry enforces (set below). if d.Policy.Mode != "off" { + meta.Value.Mode = KernelModeAudit active = append(active, d) } } @@ -313,7 +308,8 @@ func compileSlot(slot uint32, docs []Document, generation uint64, c *Compiled, o for k, v := range folded { keys = append(keys, k) // The META mode tells the program whether the slot enforces at - // all: an audit policy can carry an enforce rule. + // all: an audit policy can carry an enforce rule, and an enforce + // policy with only allow entries enforces nothing. if v.Rank == RankDenyEnforce { meta.Value.Mode = KernelModeEnforce } diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go index c9fc7da67b..d3766d5efb 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go @@ -169,9 +169,14 @@ func TestCompileSlotsMergesPolicies(t *testing.T) { if v := get(KernelKey{Slot: 1, Kind: KindDefault, ID: KindSyscall}); v.Rank != RankAllow { t.Fatalf("slot 1 default rank %d", v.Rank) } - if v := get(KernelKey{Slot: 1, Kind: KindMeta}); v.Generation != 4 || v.Mode != KernelModeEnforce { + // b enforces, but has only allow entries: the slot enforces nothing. + if v := get(KernelKey{Slot: 1, Kind: KindMeta}); v.Generation != 4 || v.Mode != KernelModeAudit { t.Fatalf("slot 1 meta %+v", v) } + // Slot 0 has a's enforce rule: it enforces. + if v := get(KernelKey{Slot: 0, Kind: KindMeta}); v.Mode != KernelModeEnforce { + t.Fatalf("slot 0 meta %+v", v) + } } // Policy A: audit deny mount. Policy B: enforce default deny, no mount rule. diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go index c35adfb140..897354ef64 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go @@ -171,7 +171,9 @@ func TestForkMarkFaultInjection(t *testing.T) { for _, mode := range []string{"enforce", "audit"} { t.Run(mode, func(t *testing.T) { f := newKernelFixture(t) - doc := privpolicy.Document{Version: 1, Name: "fault-" + mode, Policy: privpolicy.Policy{Mode: mode, Privilege: privpolicy.Section{Default: "allow"}}} + // One deny rule, so the slot enforces in enforce mode. + rule := privpolicy.Rule{Name: "no-modules", Action: "deny", Target: privpolicy.Target{Syscall: []string{"init_module"}}} + doc := privpolicy.Document{Version: 1, Name: "fault-" + mode, Policy: privpolicy.Policy{Mode: mode, Privilege: privpolicy.Section{Default: "allow", Rules: []privpolicy.Rule{rule}}}} compiled, err := privpolicy.CompileKernel(doc, 1) if err != nil { t.Fatal(err) From 2f51eeb68d82e4e0f212cd6898fd09da4a3e2818 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:09:23 +0200 Subject: [PATCH 18/34] docs(enforce_privilege): compat mknod and leaving the bound subtree For a 32-bit task every inode_mknod call is judged as mknod, including a unix socket bind. A task that can write to a cgroup outside its bound subtree runs there without the policy. Drop the verdict export item: the gadget has a verdict datasource. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/enforce_privilege/README.md | 12 +++++++++--- pkg/ebpf/gadgets/enforce_privilege/program.bpf.c | 4 +++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index a20dd5ebd4..ee2332784c 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -115,7 +115,9 @@ policy: `setns`, `open_tree`, `fsopen`, `fsmount`, `fspick`, `mount_setattr`) are judged at their `CAP_SYS_ADMIN` check by syscall number. For a 32-bit (compat) task the number is not mapped, so these checks fail closed: every - candidate syscall rule applies. + candidate syscall rule applies. The same holds for `mknod`: for a compat + task every `inode_mknod` call is judged as `mknod`, including a unix socket + `bind(2)`, so a `mknod` deny or a default deny also refuses that bind. - **Deny** returns `-EPERM` from the LSM hook, before the side effect. ## Known limits @@ -128,8 +130,12 @@ policy: - **Fault injection.** The `arming-fail-fork-marks` parameter makes every child mark fail at fork. It exists for tests (`runtime/runtime_kernel_test.go`); owners do not set it. -- **Not in this gadget yet:** export of verdicts as events (the owner logs - them), and kernels older than 6.1. +- **Kernels older than 6.1** are not supported yet. +- **Leaving the bound cgroup subtree.** A task that can write to a cgroup + outside its bound subtree (its `cgroup.procs`, or `clone3` with + `CLONE_INTO_CGROUP`) runs there without the policy. This needs write access + to the host cgroup tree, which a normal container does not have. A + `CLONE_INTO_CGROUP` clone is refused only when the child's mark fails. - **No-audit capability checks** (`CAP_OPT_NOAUDIT`) are excluded from capability enforcement. Most are kernel probes (for example the overcommit check), and judging them would deny normal work in every bound container. Some diff --git a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c index c7f894dc3c..5c5dd642ad 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c +++ b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c @@ -905,7 +905,9 @@ int BPF_PROG(ig_priv_key_permission) // vfs_mknod also runs for bind(2) of a unix socket path and for overlayfs // whiteouts (unlink or rename of a lower file). Judge only mknod(2) and -// mknodat(2). +// mknodat(2). A 32-bit (compat) task's syscall number is not mapped, so for +// it every call is judged as mknod (fail closed), its unix socket bind(2) +// included. SEC("lsm/inode_mknod") int BPF_PROG(ig_priv_mknod) { From 95c2f23f0597057e0a7041f8e8897b584083112a Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 20:10:55 +0200 Subject: [PATCH 19/34] fix(enforcement): arm a thread whose group leader has exited On kernels without PIDFD_THREAD, a leader that exits while its threads run is a zombie: its mount namespace cannot be read, so the walk took its threads for exited and left them unarmed. All threads share one mount namespace, so the leader check is dropped; pidfd_open works on a zombie leader. A thread that still runs but cannot be armed now fails the walk. The README says which failed marks the counter counts. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- .../gadgets/common/enforcement/arm_linux.go | 26 +++++++++---------- .../common/enforcement/arm_linux_test.go | 11 +++----- pkg/ebpf/gadgets/enforce_privilege/README.md | 6 +++-- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/pkg/ebpf/gadgets/common/enforcement/arm_linux.go b/pkg/ebpf/gadgets/common/enforcement/arm_linux.go index bf7ac52d57..ce32d3a1f2 100644 --- a/pkg/ebpf/gadgets/common/enforcement/arm_linux.go +++ b/pkg/ebpf/gadgets/common/enforcement/arm_linux.go @@ -110,10 +110,15 @@ func armThreads(armed Map, threads string, hostMntns uint64, seen map[int]bool) // A non-leader thread on a kernel without PIDFD_THREAD: the // BPF rule judges it through its group leader, which can be // outside this cgroup, so arm the leader. - fd, err = openLeader(scanner.Text(), hostMntns) + fd, err = openLeader(scanner.Text()) } if errors.Is(err, unix.ESRCH) || errors.Is(err, fs.ErrNotExist) { - seen[tid] = true // exited + // Exited, unless the thread still runs: then it cannot be + // armed, and the caller must not report it protected. + if _, serr := os.Stat(filepath.Join(host.HostProcFs, scanner.Text())); serr == nil { + return fmt.Errorf("task %d runs but cannot be armed: %w", tid, err) + } + seen[tid] = true continue } if err != nil { @@ -129,10 +134,12 @@ func armThreads(armed Map, threads string, hostMntns uint64, seen map[int]bool) return scanner.Err() } -// openLeader opens a pidfd of the thread group leader of tid. A leader in the -// runtime mount namespace is a runtime helper: arming it would judge the -// runtime, so the thread cannot be armed and the caller gets an error. -func openLeader(tid string, hostMntns uint64) (int, error) { +// openLeader opens a pidfd of the thread group leader of tid. All threads of a +// process share one mount namespace (setns and unshare of a mount namespace +// fail in a multithreaded process), so the caller's check of the thread's +// namespace also covers the leader. A leader that has exited while other +// threads run is a zombie; pidfd_open still works on it. +func openLeader(tid string) (int, error) { data, err := os.ReadFile(filepath.Join(host.HostProcFs, tid, "status")) if err != nil { return -1, err @@ -149,13 +156,6 @@ func openLeader(tid string, hostMntns uint64) (int, error) { if tgid <= 0 { return -1, fmt.Errorf("no leader for task %s", tid) } - mntns, err := mntnsOf(strconv.Itoa(tgid)) - if err != nil { - return -1, err - } - if mntns == hostMntns { - return -1, fmt.Errorf("thread %s has a runtime helper as leader", tid) - } return unix.PidfdOpen(tgid, 0) } diff --git a/pkg/ebpf/gadgets/common/enforcement/arm_linux_test.go b/pkg/ebpf/gadgets/common/enforcement/arm_linux_test.go index 8e5b78c27f..0b23c91d55 100644 --- a/pkg/ebpf/gadgets/common/enforcement/arm_linux_test.go +++ b/pkg/ebpf/gadgets/common/enforcement/arm_linux_test.go @@ -26,17 +26,12 @@ func TestOpenLeaderOfThread(t *testing.T) { if thread == "" { t.Skip("no second thread") } - fd, err := openLeader(thread, 0) + fd, err := openLeader(thread) if err != nil { t.Fatalf("leader of thread %s: %v", thread, err) } unix.Close(fd) - own, err := mntnsOf(pid) - if err != nil { - t.Fatal(err) - } - if fd, err := openLeader(thread, own); err == nil { - unix.Close(fd) - t.Fatal("a leader in the runtime mount namespace must not be armed") + if _, err := openLeader("999999999"); err == nil { + t.Fatal("a task that does not exist has no leader") } } diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index ee2332784c..fdfa524bef 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -98,8 +98,10 @@ policy: (`CLONE_INTO_CGROUP`) is refused the same way. An audit-only slot, or an unbound cgroup, allows the fork. A new thread of an armed group leader is covered through the leader, so it is never refused. An unarmed parent (for - example a runtime helper) is out of scope, not a failure. Every failed mark is - counted in `counters` slot 4. Moving a container to another slot arms its + example a runtime helper) is out of scope, not a failure. Every failed mark that + leaves a task unarmed is counted in `counters` slot 4. A failed mark of a new + thread is not counted: the thread is covered through its leader, and it is + marked at its own exec. Moving a container to another slot arms its running tasks again. - **Tracking faults.** A failed mark in an enforcing slot (a refused fork, or a lost mark at exec, which cannot be refused) is also counted per slot in From b09cddf195bc6312b02de3e20a1860219a9b72fb Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Thu, 5 Nov 2026 03:04:33 +0200 Subject: [PATCH 20/34] test(enforce_privilege): another BPF LSM deny holds under the gadget A kernel test attaches a program that denies chroot with EACCES, first older and then newer than the gadget, and checks that the gadget's allow never replaces that deny, for an unbound container and for a bound one with an allow rule. The kernel stops at the first nonzero BPF LSM return, so the hooks do not read the previous verdict. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- .../gadgets/enforce_privilege/program.bpf.c | 8 +- .../runtime/runtime_kernel_test.go | 80 ++++++++++++++++++- 2 files changed, 84 insertions(+), 4 deletions(-) diff --git a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c index 5c5dd642ad..9d1f55ba18 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c +++ b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c @@ -11,9 +11,11 @@ // - Only armed tasks are judged (see arming.h). // - A deny returns -EPERM from the LSM hook, before the side effect. // - The kernel calls the next BPF LSM program only while every earlier one -// returned 0, so the hooks do not read the previous return value. They also -// name only the hook arguments they need, because the trailing arguments of -// some hooks changed between kernel versions. +// returned 0 (the trampoline stops at the first nonzero return), so a later +// program cannot replace an earlier deny, and the hooks do not read the +// previous return value (runtime_kernel_test.go, TestEarlierBPFDenyHolds). +// They also name only the hook arguments they need, so a hook that gains a +// trailing argument in a newer kernel does not move what they read. // - Each hook body is a plain function, so it can also be attached as // fmod_ret on the matching security_* function. diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go index 897354ef64..2c01525761 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go @@ -17,7 +17,8 @@ package runtime // PRIVILEGE_CGROUP_PATH its cgroup v2 path under /sys/fs/cgroup, with a leading "/" (as in /proc//cgroup) // // TestForkMarkFaultInjection uses the same fixture. The container needs a -// POSIX sh. +// POSIX sh. TestEarlierBPFDenyHolds loads a BPF LSM program that denies every +// chroot on the node for a few seconds. import ( "context" @@ -25,10 +26,14 @@ import ( "os/exec" "path/filepath" "strconv" + "strings" "sync/atomic" "testing" "time" + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/asm" + "github.com/cilium/ebpf/link" "github.com/inspektor-gadget/inspektor-gadget/pkg/datasource" gadgetcontext "github.com/inspektor-gadget/inspektor-gadget/pkg/gadget-context" "github.com/inspektor-gadget/inspektor-gadget/pkg/gadget-service/api" @@ -211,3 +216,76 @@ func TestForkMarkFaultInjection(t *testing.T) { }) } } + +// TestEarlierBPFDenyHolds checks that the gadget never turns another BPF LSM +// program's deny into an allow, in both attach orders. The kernel runs the +// programs of a hook newest first and stops at the first nonzero return. The +// other program denies chroot with EACCES; the gadget would deny with EPERM, +// so the errno shows which program decided. +func TestEarlierBPFDenyHolds(t *testing.T) { + f := newKernelFixture(t) + chroot := func() string { + out, err := exec.CommandContext(f.ctx, "docker", "exec", f.id, "chroot", "/", "true").CombinedOutput() + switch { + case err == nil: + return "ok" + case strings.Contains(string(out), "Permission denied"): + return "EACCES" + case strings.Contains(string(out), "Operation not permitted"): + return "EPERM" + } + return string(out) + } + prog, err := ebpf.NewProgram(&ebpf.ProgramSpec{ + Type: ebpf.LSM, AttachType: ebpf.AttachLSMMac, AttachTo: "path_chroot", License: "GPL", + Instructions: asm.Instructions{asm.Mov.Imm(asm.R0, -13), asm.Return()}, // -EACCES + }) + if err != nil { + t.Fatal(err) + } + defer prog.Close() + attach := func() link.Link { + l, err := link.AttachLSM(link.LSMOptions{Program: prog}) + if err != nil { + t.Fatal(err) + } + return l + } + if got := chroot(); got != "ok" { + t.Fatalf("baseline chroot: %s", got) + } + older := attach() + defer older.Close() + if got := chroot(); got != "EACCES" { + t.Fatalf("the other program must deny chroot: %s", got) + } + rule := privpolicy.Rule{Name: "chroot-ok", Action: "allow", Target: privpolicy.Target{Syscall: []string{"chroot"}}} + doc := privpolicy.Document{Version: 1, Name: "allow-chroot", Policy: privpolicy.Policy{Mode: "enforce", Privilege: privpolicy.Section{Default: "deny", Rules: []privpolicy.Rule{rule}}}} + compiled, err := privpolicy.CompileKernel(doc, 1) + if err != nil { + t.Fatal(err) + } + f.start(t, "privilege-earlier-deny", compiled, nil) + if got := chroot(); got != "EACCES" { + t.Fatalf("unbound, gadget newer: %s", got) + } + if err := f.bindings.Bind(enforcement.Binding{Container: f.container}); err != nil { + t.Fatal(err) + } + defer f.bindings.Unbind(f.container) + // The gadget runs first and allows; the older program still denies. + if got := chroot(); got != "EACCES" { + t.Fatalf("bound with an allow rule, gadget newer: %s", got) + } + older.Close() + if got := chroot(); got != "ok" { + t.Fatalf("the gadget alone allows chroot: %s", got) + } + // The other program is now newer: it runs first and denies, and the + // gadget's allow does not replace that. + newer := attach() + defer newer.Close() + if got := chroot(); got != "EACCES" { + t.Fatalf("bound with an allow rule, other program newer: %s", got) + } +} From 0e012a21b1ce6b76204163a1a088b0347cd3b91d Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Thu, 5 Nov 2026 10:48:25 +0200 Subject: [PATCH 21/34] fix(enforce_privilege): return an earlier BPF verdict unchanged Every int LSM hook now starts with "if (ret) return ret;", as the other enforcement gadgets do. The kernel already stops at the first nonzero BPF LSM return; this keeps the rule explicit. ret follows the hook arguments, so the operator checks every hook argument count in the kernel BTF before the programs load (enforcement.CheckLSMHookArgs) and refuses a kernel where one differs. The counts are equal on 6.8 and 6.12. A unit test keeps runtime.Hooks in step with the program. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- .../gadgets/common/enforcement/lsmhooks.go | 29 +++++ .../common/enforcement/lsmhooks_test.go | 40 +++++++ .../gadgets/enforce_privilege/program.bpf.c | 106 +++++++++++++----- .../enforce_privilege/runtime/runtime.go | 28 +++++ .../enforce_privilege/runtime/runtime_test.go | 29 +++++ 5 files changed, 202 insertions(+), 30 deletions(-) create mode 100644 pkg/ebpf/gadgets/common/enforcement/lsmhooks.go create mode 100644 pkg/ebpf/gadgets/common/enforcement/lsmhooks_test.go diff --git a/pkg/ebpf/gadgets/common/enforcement/lsmhooks.go b/pkg/ebpf/gadgets/common/enforcement/lsmhooks.go new file mode 100644 index 0000000000..5bdec1263a --- /dev/null +++ b/pkg/ebpf/gadgets/common/enforcement/lsmhooks.go @@ -0,0 +1,29 @@ +// Copyright 2026 The Kubescape Authors +// SPDX-License-Identifier: Apache-2.0 + +package enforcement + +import ( + "fmt" + + "github.com/cilium/ebpf/btf" +) + +// CheckLSMHookArgs checks the argument count of each BPF LSM hook (name -> +// count) in the kernel's BTF. A BPF LSM program reads the previous BPF verdict, +// `ret`, right after the hook arguments, so a hook with another argument count +// would make it read an argument as the verdict. Call it before the programs +// are loaded, and refuse the kernel on an error. +func CheckLSMHookArgs(kernel *btf.Spec, hooks map[string]int) error { + for hook, args := range hooks { + var fn *btf.Func + if err := kernel.TypeByName("bpf_lsm_"+hook, &fn); err != nil { + return fmt.Errorf("LSM hook %s: %w", hook, err) + } + proto, ok := fn.Type.(*btf.FuncProto) + if !ok || len(proto.Params) != args { + return fmt.Errorf("LSM hook %s has an unexpected signature; this kernel is not supported", hook) + } + } + return nil +} diff --git a/pkg/ebpf/gadgets/common/enforcement/lsmhooks_test.go b/pkg/ebpf/gadgets/common/enforcement/lsmhooks_test.go new file mode 100644 index 0000000000..e2f56914d0 --- /dev/null +++ b/pkg/ebpf/gadgets/common/enforcement/lsmhooks_test.go @@ -0,0 +1,40 @@ +// Copyright 2026 The Kubescape Authors +// SPDX-License-Identifier: Apache-2.0 + +package enforcement + +import ( + "bytes" + "testing" + + "github.com/cilium/ebpf/btf" +) + +func TestCheckLSMHookArgs(t *testing.T) { + b, err := btf.NewBuilder(nil, nil) + if err != nil { + t.Fatal(err) + } + ptr := &btf.Pointer{Target: &btf.Void{}} + proto := &btf.FuncProto{Return: &btf.Int{Name: "int", Size: 4, Encoding: btf.Signed}, Params: []btf.FuncParam{{Name: "a", Type: ptr}, {Name: "b", Type: ptr}}} + if _, err := b.Add(&btf.Func{Name: "bpf_lsm_two", Type: proto, Linkage: btf.GlobalFunc}); err != nil { + t.Fatal(err) + } + raw, err := b.Marshal(nil, nil) + if err != nil { + t.Fatal(err) + } + kernel, err := btf.LoadSpecFromReader(bytes.NewReader(raw)) + if err != nil { + t.Fatal(err) + } + if err := CheckLSMHookArgs(kernel, map[string]int{"two": 2}); err != nil { + t.Fatalf("matching count: %v", err) + } + if CheckLSMHookArgs(kernel, map[string]int{"two": 3}) == nil { + t.Fatal("another argument count must be refused") + } + if CheckLSMHookArgs(kernel, map[string]int{"missing": 1}) == nil { + t.Fatal("a missing hook must be refused") + } +} diff --git a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c index 9d1f55ba18..6ceca3b5b8 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c +++ b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c @@ -11,11 +11,13 @@ // - Only armed tasks are judged (see arming.h). // - A deny returns -EPERM from the LSM hook, before the side effect. // - The kernel calls the next BPF LSM program only while every earlier one -// returned 0 (the trampoline stops at the first nonzero return), so a later -// program cannot replace an earlier deny, and the hooks do not read the -// previous return value (runtime_kernel_test.go, TestEarlierBPFDenyHolds). -// They also name only the hook arguments they need, so a hook that gains a -// trailing argument in a newer kernel does not move what they read. +// returned 0 (the trampoline runs the newest program first and stops at the +// first nonzero return; runtime_kernel_test.go, TestEarlierBPFDenyHolds). +// Each hook also returns an earlier BPF verdict, `ret`, unchanged, as the +// other enforcement gadgets do. `ret` follows the hook arguments, so its +// position depends on the hook signature: the operator checks every +// argument count in the kernel's BTF (runtime.Hooks) and refuses a kernel +// where one differs. // - Each hook body is a plain function, so it can also be attached as // fmod_ret on the matching security_* function. @@ -542,8 +544,10 @@ static __always_inline int do_userns_create(void *ctx) } SEC("lsm/userns_create") -int BPF_PROG(ig_priv_userns, const struct cred *cred) +int BPF_PROG(ig_priv_userns, const struct cred *cred, int ret) { + if (ret) + return ret; return do_userns_create(ctx); } @@ -585,9 +589,10 @@ static __always_inline int do_task_alloc(void *ctx, struct task_struct *task, } SEC("lsm/task_alloc") -int BPF_PROG(ig_priv_task_alloc, struct task_struct *task, - unsigned long clone_flags) +int BPF_PROG(ig_priv_task_alloc, struct task_struct *task, unsigned long clone_flags, int ret) { + if (ret) + return ret; return do_task_alloc(ctx, task, clone_flags); } @@ -666,35 +671,44 @@ static __always_inline int do_capable(void *ctx, const struct cred *cred, } SEC("lsm/capable") -int BPF_PROG(ig_priv_capable, const struct cred *cred, - struct user_namespace *ns, int cap, unsigned int opts) +int BPF_PROG(ig_priv_capable, const struct cred *cred, struct user_namespace *ns, int cap, unsigned int opts, int ret) { + if (ret) + return ret; return do_capable(ctx, cred, cap, opts); } SEC("lsm/capset") -int BPF_PROG(ig_priv_capset) +int BPF_PROG(ig_priv_capset, struct cred *new, const struct cred *old, const kernel_cap_t *effective, const kernel_cap_t *inheritable, const kernel_cap_t *permitted, int ret) { + if (ret) + return ret; return check_syscall(ctx, PRIV_HOOK_CAPSET, PRIV_SC_CAPSET, 0); } SEC("lsm/task_fix_setuid") -int BPF_PROG(ig_priv_setuid) +int BPF_PROG(ig_priv_setuid, struct cred *new, const struct cred *old, int flags, int ret) { + if (ret) + return ret; return check_syscall(ctx, PRIV_HOOK_TASK_FIX_SETUID, PRIV_SC_SETUID, 0); } SEC("lsm/task_fix_setgid") -int BPF_PROG(ig_priv_setgid) +int BPF_PROG(ig_priv_setgid, struct cred *new, const struct cred *old, int flags, int ret) { + if (ret) + return ret; return check_syscall(ctx, PRIV_HOOK_TASK_FIX_SETGID, PRIV_SC_SETGID, 0); } /* Mounts */ SEC("lsm/sb_mount") -int BPF_PROG(ig_priv_sb_mount) +int BPF_PROG(ig_priv_sb_mount, const char *dev_name, const struct path *path, const char *type, unsigned long flags, void *data, int ret) { + if (ret) + return ret; return check_syscall(ctx, PRIV_HOOK_SB_MOUNT, PRIV_SC_MOUNT, 0); } @@ -720,33 +734,43 @@ static __always_inline int do_sb_kern_mount(void *ctx, } SEC("lsm/sb_kern_mount") -int BPF_PROG(ig_priv_sb_kern_mount, const struct super_block *sb) +int BPF_PROG(ig_priv_sb_kern_mount, const struct super_block *sb, int ret) { + if (ret) + return ret; return do_sb_kern_mount(ctx, sb); } SEC("lsm/sb_umount") -int BPF_PROG(ig_priv_sb_umount) +int BPF_PROG(ig_priv_sb_umount, struct vfsmount *mnt, int flags, int ret) { + if (ret) + return ret; return check_syscall(ctx, PRIV_HOOK_SB_UMOUNT, PRIV_SC_UMOUNT2, 0); } SEC("lsm/move_mount") -int BPF_PROG(ig_priv_move_mount) +int BPF_PROG(ig_priv_move_mount, const struct path *from_path, const struct path *to_path, int ret) { + if (ret) + return ret; return check_syscall(ctx, PRIV_HOOK_MOVE_MOUNT, PRIV_SC_MOVE_MOUNT, 0); } SEC("lsm/sb_pivotroot") -int BPF_PROG(ig_priv_pivotroot) +int BPF_PROG(ig_priv_pivotroot, const struct path *old_path, const struct path *new_path, int ret) { + if (ret) + return ret; return check_syscall(ctx, PRIV_HOOK_SB_PIVOTROOT, PRIV_SC_PIVOT_ROOT, 0); } SEC("lsm/path_chroot") -int BPF_PROG(ig_priv_chroot) +int BPF_PROG(ig_priv_chroot, const struct path *path, int ret) { + if (ret) + return ret; return check_syscall(ctx, PRIV_HOOK_PATH_CHROOT, PRIV_SC_CHROOT, 0); } @@ -771,8 +795,10 @@ static __always_inline int do_anon_inode(void *ctx, const struct qstr *name) } SEC("lsm/inode_init_security_anon") -int BPF_PROG(ig_priv_anon_inode, struct inode *inode, const struct qstr *name) +int BPF_PROG(ig_priv_anon_inode, struct inode *inode, const struct qstr *name, const struct inode *context_inode, int ret) { + if (ret) + return ret; return do_anon_inode(ctx, name); } @@ -818,14 +844,18 @@ static __always_inline int do_ptrace_access_check(void *ctx, unsigned int mode) } SEC("lsm/ptrace_access_check") -int BPF_PROG(ig_priv_ptrace, struct task_struct *child, unsigned int mode) +int BPF_PROG(ig_priv_ptrace, struct task_struct *child, unsigned int mode, int ret) { + if (ret) + return ret; return do_ptrace_access_check(ctx, mode); } SEC("lsm/ptrace_traceme") -int BPF_PROG(ig_priv_traceme) +int BPF_PROG(ig_priv_traceme, struct task_struct *parent, int ret) { + if (ret) + return ret; return check_syscall(ctx, PRIV_HOOK_PTRACE_TRACEME, PRIV_SC_PTRACE, 0); } @@ -846,8 +876,10 @@ static __always_inline int do_kernel_read_file(void *ctx, int id) } SEC("lsm/kernel_read_file") -int BPF_PROG(ig_priv_kernel_read_file, struct file *file, int id) +int BPF_PROG(ig_priv_kernel_read_file, struct file *file, int id, bool contents, int ret) { + if (ret) + return ret; return do_kernel_read_file(ctx, id); } @@ -864,16 +896,20 @@ static __always_inline int do_kernel_load_data(void *ctx, int id) } SEC("lsm/kernel_load_data") -int BPF_PROG(ig_priv_kernel_load_data, int id) +int BPF_PROG(ig_priv_kernel_load_data, int id, bool contents, int ret) { + if (ret) + return ret; return do_kernel_load_data(ctx, id); } // An implicit module load requested on behalf of the task (for example an // unknown socket family). It is judged as init_module. SEC("lsm/kernel_module_request") -int BPF_PROG(ig_priv_module_request) +int BPF_PROG(ig_priv_module_request, char *kmod_name, int ret) { + if (ret) + return ret; return check_syscall(ctx, PRIV_HOOK_KERNEL_MODULE_REQUEST, PRIV_SC_INIT_MODULE, 0); } @@ -881,27 +917,35 @@ int BPF_PROG(ig_priv_module_request) /* bpf, perf, keys, mknod */ SEC("lsm/bpf") -int BPF_PROG(ig_priv_bpf) +int BPF_PROG(ig_priv_bpf, int cmd, union bpf_attr *attr, unsigned int size, int ret) { + if (ret) + return ret; return check_syscall(ctx, PRIV_HOOK_BPF, PRIV_SC_BPF, 0); } SEC("lsm/perf_event_open") -int BPF_PROG(ig_priv_perf_event_open) +int BPF_PROG(ig_priv_perf_event_open, struct perf_event_attr *attr, int type, int ret) { + if (ret) + return ret; return check_syscall(ctx, PRIV_HOOK_PERF_EVENT_OPEN, PRIV_SC_PERF_EVENT_OPEN, 0); } SEC("lsm/key_alloc") -int BPF_PROG(ig_priv_key_alloc) +int BPF_PROG(ig_priv_key_alloc, struct key *key, const struct cred *cred, unsigned long flags, int ret) { + if (ret) + return ret; return check_syscall(ctx, PRIV_HOOK_KEY_ALLOC, PRIV_SC_KEYCTL, 0); } SEC("lsm/key_permission") -int BPF_PROG(ig_priv_key_permission) +int BPF_PROG(ig_priv_key_permission, key_ref_t key_ref, const struct cred *cred, unsigned int need_perm, int ret) { + if (ret) + return ret; return check_syscall(ctx, PRIV_HOOK_KEY_PERMISSION, PRIV_SC_KEYCTL, 0); } @@ -911,8 +955,10 @@ int BPF_PROG(ig_priv_key_permission) // it every call is judged as mknod (fail closed), its unix socket bind(2) // included. SEC("lsm/inode_mknod") -int BPF_PROG(ig_priv_mknod) +int BPF_PROG(ig_priv_mknod, struct inode *dir, struct dentry *dentry, umode_t mode, unsigned int dev, int ret) { + if (ret) + return ret; if (!current_is_compat() && priv_syscall_from_nr(current_syscall_nr()) != PRIV_SC_MKNOD) return 0; diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go index c38d7d732e..72ea45e68e 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go @@ -10,6 +10,7 @@ import ( "fmt" "github.com/cilium/ebpf" + "github.com/cilium/ebpf/btf" "github.com/spf13/viper" "golang.org/x/sys/unix" @@ -30,6 +31,28 @@ const ( type ContainerIdentity = enforcement.ContainerIdentity +// Hooks lists the int LSM hooks of ../program.bpf.c with their argument count +// (without `ret`). Each program reads the previous verdict, `ret`, right after +// them, so the counts must match the running kernel. +var Hooks = map[string]int{ + "userns_create": 1, "task_alloc": 2, "capable": 4, "capset": 5, + "task_fix_setuid": 3, "task_fix_setgid": 3, "sb_mount": 5, "sb_kern_mount": 1, + "sb_umount": 2, "move_mount": 2, "sb_pivotroot": 2, "path_chroot": 1, + "inode_init_security_anon": 3, "ptrace_access_check": 2, "ptrace_traceme": 1, + "kernel_read_file": 3, "kernel_load_data": 2, "kernel_module_request": 1, + "bpf": 3, "perf_event_open": 2, "key_alloc": 3, "key_permission": 3, "inode_mknod": 4, +} + +// checkKernel refuses a kernel whose hook signatures differ from Hooks +// (replaced in tests). +var checkKernel = func() error { + kernel, err := btf.LoadKernelSpec() + if err != nil { + return fmt.Errorf("kernel BTF: %w", err) + } + return enforcement.CheckLSMHookArgs(kernel, Hooks) +} + // Config holds the compiled rules and the initial container selection. type Config struct { Policy privpolicy.Compiled @@ -118,6 +141,11 @@ func (*operator) InstantiateDataOperator(gc operators.GadgetContext, _ api.Param if err := validateConfig(config); err != nil { return nil, err } + // Before the programs are loaded: a wrong argument count would make them + // read an argument as the previous verdict. + if err := checkKernel(); err != nil { + return nil, fmt.Errorf("enforce_privilege: %w", err) + } b, err := enforcement.New(enforcement.Config{ Name: operatorName, Generation: config.Policy.Generation, diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go index 97a251a2c7..9c3c5a63c5 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go @@ -4,6 +4,9 @@ package runtime import ( + "os" + "regexp" + "strings" "testing" "github.com/cilium/ebpf" @@ -67,3 +70,29 @@ func TestValidateConfig(t *testing.T) { t.Fatal("duplicate generation entries must fail") } } + +// TestHooksMatchProgram keeps Hooks in step with the BPF_PROG signatures: every +// int hook ends with `int ret`, after the listed number of arguments. +func TestHooksMatchProgram(t *testing.T) { + data, err := os.ReadFile("../program.bpf.c") + if err != nil { + t.Fatal(err) + } + programs := regexp.MustCompile(`SEC\("lsm/(\w+)"\)\s*\nint BPF_PROG\(\w+((?:,[^)]*)?)\)`).FindAllStringSubmatch(string(data), -1) + seen := map[string]bool{} + for _, m := range programs { + if m[1] == "bprm_committed_creds" { // void hook: no verdict + continue + } + seen[m[1]] = true + if !strings.HasSuffix(strings.TrimSpace(m[2]), "int ret") { + t.Errorf("hook %s does not end with int ret", m[1]) + } + if args := strings.Count(m[2], ",") - 1; Hooks[m[1]] != args { + t.Errorf("hook %s: program has %d arguments, Hooks has %d", m[1], args, Hooks[m[1]]) + } + } + if len(seen) != len(Hooks) { + t.Errorf("program has %d int hooks, Hooks has %d", len(seen), len(Hooks)) + } +} From ccd520223fd995367b4ec360c1a628adc43c4060 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:07:41 +0200 Subject: [PATCH 22/34] fix(enforcement): do not arm OCI hooks that runc init forks in the container A parent cgroup bound before its containers start (a pod cgroup) also covered the OCI createContainer and startContainer hooks: runc init forks them inside the container mount namespace, so they were armed and judged as workload. A strict policy then stopped the container from starting (for example kind's own createContainer hook, a bind mount). A task is no longer armed at exec when its parent is unarmed, outside the runtime mount namespace, and in the same mount namespace and cgroup as the task: it was forked by a runtime helper inside the container. The workload and exec commands are execed by runc init itself, whose parent is in the runtime mount namespace, so they stay armed, and their children inherit the mark as before. An unknown runtime namespace trusts no parent. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/common/README.md | 2 +- pkg/ebpf/gadgets/common/arming.h | 37 ++- pkg/ebpf/gadgets/enforce_privilege/README.md | 13 ++ .../runtime/runtime_kernel_test.go | 211 ++++++++++++++++++ 4 files changed, 261 insertions(+), 2 deletions(-) diff --git a/pkg/ebpf/gadgets/common/README.md b/pkg/ebpf/gadgets/common/README.md index cce63570ee..6e50274597 100644 --- a/pkg/ebpf/gadgets/common/README.md +++ b/pkg/ebpf/gadgets/common/README.md @@ -17,7 +17,7 @@ and keeps its path rules in an array that `policy.h` does not describe. | `bindings.h` | `bindings` map: cgroup id → the container's policy slot; optional parent-cgroup depth | | `policy.h` | policy modes (0 off, 1 audit, 2 enforce), rank resolution (explicit deny > allow > default, one mode per rule), shared policy value | | `verdict.h` | first verdict fields (`generation`, `cgroup`, `deny`, `mode`), per-CPU `counters` (denied, submitted, dropped, allowed, section error, rate limited), 100 reports per CPU per second | -| `arming.h` | arming rule: judge only tasks that executed outside the runtime mount namespace, and their children; `arming_faults`, tracking faults per slot | +| `arming.h` | arming rule: judge only tasks that executed outside the runtime mount namespace (not those forked there by an unarmed runtime helper, such as OCI hooks), and their children; `arming_faults`, tracking faults per slot | | `merge/` | the merge contract for several policies on one container, and its shared test vectors | | `enforcement/` | Go: the binding controller of one node-wide instance (policy written and frozen before any binding, live `Bind`/`Unbind` of a container to a slot, readiness, arming of tasks already running, `SlotFaults`) | diff --git a/pkg/ebpf/gadgets/common/arming.h b/pkg/ebpf/gadgets/common/arming.h index 029a56a216..b7bf0f222d 100644 --- a/pkg/ebpf/gadgets/common/arming.h +++ b/pkg/ebpf/gadgets/common/arming.h @@ -17,6 +17,15 @@ // runc init for "exec" execs inside the container cgroup but still in the host // mount namespace, so it is not armed. The command it execs after joining the // container mount namespace is armed. +// +// OCI createContainer and startContainer hooks are runtime setup too: runc init +// forks them inside the container mount namespace and cgroup, before the +// workload starts. So a task is not armed at exec when its parent is unarmed, +// outside the host mount namespace, and in the same mount namespace and cgroup +// as the task (a runtime helper inside the container). The workload and an +// "exec" command are execed by runc init itself, whose parent (the runtime or +// its shim) is in the host mount namespace. Such a helper, and a hook, are +// trusted not to leave a process behind: its children stay unarmed. #ifndef __ENFORCEMENT_ARMING_H #define __ENFORCEMENT_ARMING_H @@ -126,16 +135,42 @@ static __always_inline __u64 arming_host_mntns(struct task_struct *task) return 0; } +// arming_runtime_child reports whether the task was forked by a runtime +// helper inside the container (an OCI hook, or a hook's child): its parent is +// unarmed, outside the host mount namespace, and in the same mount namespace +// and cgroup as the task. A parent with no mount namespace (exiting) does not +// match, so the task is armed. The parent pointer is a direct BTF access (not +// CO-RE): task storage needs a trusted task pointer. +static __always_inline bool arming_runtime_child(struct task_struct *task, + __u64 mntns, __u64 host) +{ + struct task_struct *parent = task->real_parent; + __u64 parent_mntns; + + if (!parent || arming_task_armed(parent)) + return false; + parent_mntns = BPF_CORE_READ(parent, nsproxy, mnt_ns, ns.inum); + if (!parent_mntns || parent_mntns == host || parent_mntns != mntns) + return false; + return BPF_CORE_READ(parent, cgroups, dfl_cgrp, kn, id) == + BPF_CORE_READ(task, cgroups, dfl_cgrp, kn, id); +} + // arming_on_exec runs after a successful exec (bprm_committed_creds). // arming_on_exec returns false when a needed mark failed. static __always_inline bool arming_on_exec(void) { struct task_struct *task = bpf_get_current_task_btf(); __u64 mntns = BPF_CORE_READ(task, nsproxy, mnt_ns, ns.inum); + __u64 host; if (arming_task_armed(task)) return true; - if (mntns == arming_host_mntns(task)) + host = arming_host_mntns(task); + if (mntns == host) + return true; + // An unknown host namespace trusts no parent. + if (host && arming_runtime_child(task, mntns, host)) return true; return arming_mark(task); } diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index fdfa524bef..45ec570175 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -86,6 +86,10 @@ policy: program outside the container runtime's mount namespace; children inherit the mark. So runc's own setup (namespaces, mounts, `pivot_root`, `setns` for `exec`) is never blocked, and `docker run` / `docker exec` keep working. + OCI `createContainer` and `startContainer` hooks are not armed either: a + task is not armed at exec when its parent is unarmed, outside the runtime + mount namespace, and in the same mount namespace and cgroup (runc init forks + the hooks inside the container). When a container is bound, the owner also arms the tasks that already run in it (pidfd writes to `armed_tasks`). The runtime mount namespace is the `host-mntns-id` gadget parameter; the owner sets it to the mount namespace of @@ -124,6 +128,15 @@ policy: ## Known limits +- **Runtime helpers are trusted.** A task forked by an unarmed task inside the + container (an OCI hook, or a process a hook leaves behind) is not judged, + and neither are its children. Only the runtime or a host root process can + create such a task. This was verified with runc, which execs the workload + and `exec` commands from runc init itself; a runtime that forks them from a + helper inside the container would leave them unjudged. A binding (or + owner restart) between `runc create` and `runc start` arms runc init, and + the startContainer hooks it then runs are judged. + - **Lost mark at exec.** The exec hook cannot fail the exec, so a task whose mark fails there stays unarmed until its next exec. In an enforcing slot this is a tracking fault (see above). A later check cannot tell such a task apart diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go index 2c01525761..124314071e 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go @@ -22,6 +22,8 @@ package runtime import ( "context" + "encoding/json" + "fmt" "os" "os/exec" "path/filepath" @@ -289,3 +291,212 @@ func TestEarlierBPFDenyHolds(t *testing.T) { t.Fatalf("bound with an allow rule, other program newer: %s", got) } } + +// hookFixture is a runc bundle whose OCI hooks run chroot, and a parent cgroup +// (the cgroup of a group of containers) that is bound before the container +// starts, with a policy that denies chroot. +type hookFixture struct { + *kernelFixture + bundle, parent, id string + run func(string, ...string) (string, error) +} + +// newHookFixture needs runc, PRIVILEGE_ROOTFS (a rootfs folder with busybox sh +// and chroot; it is copied) and util-linux nsenter on the host. extra are ebpf +// gadget params. +func newHookFixture(t *testing.T, extra map[string]string) *hookFixture { + t.Helper() + if os.Getenv("PRIVILEGE_KERNEL_TEST") != "1" || os.Getenv("PRIVILEGE_ROOTFS") == "" { + t.Skip("explicit disposable kernel fixture opt-in and PRIVILEGE_ROOTFS required") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + t.Cleanup(cancel) + h := &hookFixture{kernelFixture: &kernelFixture{ctx: ctx}, bundle: t.TempDir(), id: fmt.Sprintf("privilege-hooks-%d", os.Getpid())} + h.run = func(name string, args ...string) (string, error) { + out, err := exec.CommandContext(ctx, name, args...).CombinedOutput() + return strings.TrimSpace(string(out)), err + } + + // The parent cgroup, bound before the container exists (it is empty, so + // binding it arms no task). + h.parent = fmt.Sprintf("/privilege-hooks-test-%d", os.Getpid()) + if err := os.Mkdir(filepath.Join("/sys/fs/cgroup", h.parent), 0o755); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.Remove(filepath.Join("/sys/fs/cgroup", h.parent)) }) + var st unix.Stat_t + if err := unix.Stat(filepath.Join("/sys/fs/cgroup", h.parent), &st); err != nil { + t.Fatal(err) + } + parentID := ContainerIdentity{Runtime: "test-parent", ID: "hooks", CgroupID: st.Ino, CgroupPath: h.parent} + + // The bundle: hooks and workload all try chroot. The hooks run chroot + // as a child of their shell, so a hook's children are covered too. + if out, err := h.run("cp", "-a", os.Getenv("PRIVILEGE_ROOTFS"), filepath.Join(h.bundle, "rootfs")); err != nil { + t.Fatalf("copy rootfs: %v %s", err, out) + } + if out, err := h.run("runc", "spec", "--bundle", h.bundle); err != nil { + t.Fatalf("runc spec: %v %s", err, out) + } + var spec map[string]any + data, err := os.ReadFile(filepath.Join(h.bundle, "config.json")) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(data, &spec); err != nil { + t.Fatal(err) + } + process := spec["process"].(map[string]any) + process["terminal"] = false + process["args"] = []string{"sh", "-c", "chroot / true; echo $? > /tmp/workload-rc; sleep 60"} + caps := process["capabilities"].(map[string]any) + for _, set := range []string{"bounding", "effective", "permitted"} { + caps[set] = append(caps[set].([]any), "CAP_SYS_CHROOT", "CAP_SYS_ADMIN") + } + spec["root"].(map[string]any)["readonly"] = false + spec["linux"].(map[string]any)["cgroupsPath"] = h.parent + "/container" + hook := func(path string) []map[string]any { + return []map[string]any{{"path": path, "args": []string{"sh", "-c", "chroot / true; true"}}} + } + // createContainer runs before pivot_root (host path), startContainer after + // it (container path). + spec["hooks"] = map[string]any{"createContainer": hook("/bin/sh"), "startContainer": hook("/bin/sh")} + if data, err = json.Marshal(spec); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(h.bundle, "config.json"), data, 0o644); err != nil { + t.Fatal(err) + } + + rule := privpolicy.Rule{Name: "no-chroot", Action: "deny", Target: privpolicy.Target{Syscall: []string{"chroot"}}} + doc := privpolicy.Document{Version: 1, Name: "hooks", Policy: privpolicy.Policy{Mode: "enforce", Privilege: privpolicy.Section{Default: "allow", Rules: []privpolicy.Rule{rule}}}} + compiled, err := privpolicy.CompileKernel(doc, 1) + if err != nil { + t.Fatal(err) + } + h.start(t, "privilege-hooks", compiled, extra) + if err := h.bindings.Bind(enforcement.Binding{Container: parentID}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { h.bindings.Unbind(parentID) }) + return h +} + +// startContainer runs the bundle detached and returns runc's output. +func (h *hookFixture) startContainer(t *testing.T) (string, error) { + t.Helper() + // Output to a file, not a pipe: the container would hold a pipe open + // until it exits. + logPath := filepath.Join(h.bundle, "runc.log") + logFile, err := os.Create(logPath) + if err != nil { + t.Fatal(err) + } + start := exec.CommandContext(h.ctx, "runc", "run", "-d", "--bundle", h.bundle, h.id) + start.Stdout, start.Stderr = logFile, logFile + err = start.Run() + logFile.Close() + t.Cleanup(func() { h.run("runc", "delete", "-f", h.id) }) + out, _ := os.ReadFile(logPath) + return string(out), err +} + +// workloadDenied checks that the workload's first command, chroot, was denied. +func (h *hookFixture) workloadDenied(t *testing.T) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + var rc []byte + for time.Now().Before(deadline) { + if data, err := os.ReadFile(filepath.Join(h.bundle, "rootfs", "tmp", "workload-rc")); err == nil && len(data) > 0 { + rc = data + break + } + time.Sleep(100 * time.Millisecond) + } + if strings.TrimSpace(string(rc)) != "1" { + t.Fatalf("the workload's first chroot must be denied, rc=%q", rc) + } +} + +// TestRuntimeHooksAreNotArmed: the OCI createContainer and startContainer +// hooks (and their children) are runtime setup, so they must not be judged +// and the container must start. The workload, a "runc exec" command and every +// task they start must be judged. +func TestRuntimeHooksAreNotArmed(t *testing.T) { + h := newHookFixture(t, nil) + f, parent, id, run := h.kernelFixture, h.parent, h.id, h.run + if out, err := h.startContainer(t); err != nil { + t.Fatalf("the hooks must not be judged, so the container must start: %v %s", err, out) + } + + h.workloadDenied(t) + if out, err := run("runc", "exec", id, "sh", "-c", "unshare -m true && true"); err != nil { + t.Fatalf("an allowed exec must work: %v %s", err, out) + } + denied := func(name string, args ...string) { + t.Helper() + if out, err := run("runc", append([]string{"exec", id}, args...)...); err == nil || !strings.Contains(out, "not permitted") { + t.Errorf("%s: chroot must be denied by the policy: %v %s", name, err, out) + } + } + // An exec command, and tasks that armed tasks start: a forked child, an + // exec in place, a new mount namespace, and a "runc-like" parent that forks + // and then execs. + denied("runc exec", "chroot", "/", "true") + denied("child", "sh", "-c", "sh -c 'chroot / true'") + denied("exec in place", "sh", "-c", "exec chroot / true") + denied("new mount namespace", "unshare", "-m", "sh", "-c", "chroot / true") + denied("fork then exec", "sh", "-c", "(sleep 0.1; exec chroot / true) & wait $!") + + // A host process that joins the container: nsenter execs the command + // itself, so its parent is a host task. Moved into the bound cgroup + // first, so it is judged. + state, err := run("runc", "state", id) + if err != nil { + t.Fatal(err) + } + var s struct{ Pid int } + if err := json.Unmarshal([]byte(state), &s); err != nil { + t.Fatal(err) + } + hostCgroup := filepath.Join("/sys/fs/cgroup", parent, "host") + if err := os.Mkdir(hostCgroup, 0o755); err != nil { + t.Fatal(err) + } + defer os.Remove(hostCgroup) + nsenter := func(fork bool) string { + args := fmt.Sprintf("echo $$ > %s/cgroup.procs && exec nsenter -t %d -m", hostCgroup, s.Pid) + if fork { + args += " -p" + } + out, err := run("sh", "-c", args+" chroot / true") + return fmt.Sprintf("err=%v %s", err, out) + } + if out := nsenter(false); !strings.Contains(out, "not permitted") { + t.Errorf("a host nsenter that execs chroot in a bound cgroup must be denied: %s", out) + } + if f.verdicts.Load() == 0 { + t.Error("no verdict event for the denies") + } + // With -p, nsenter forks: the command's parent is the unarmed nsenter inside the + // container mount namespace, like a runtime helper: not judged. Only a + // host root process can do this; it is logged, not asserted. + t.Logf("host nsenter -p (forks) chroot: %s", nsenter(true)) +} + +// TestWrongRuntimeNamespaceStillJudgesWorkload: with a wrong host-mntns-id, +// the runtime's own tasks are outside the "host" mount namespace too. They +// stay unarmed because each is forked by a runtime task in the same mount +// namespace and cgroup. The workload is in the container mount namespace and +// its parent (runc) is not, so it must still be armed and judged. +func TestWrongRuntimeNamespaceStillJudgesWorkload(t *testing.T) { + h := newHookFixture(t, map[string]string{"host-mntns-id": "1"}) + if out, err := h.startContainer(t); err != nil { + t.Logf("container start with a wrong runtime namespace: %v %s", err, out) + return // judging the runtime itself fails closed + } + h.workloadDenied(t) + if out, err := h.run("runc", "exec", h.id, "chroot", "/", "true"); err == nil || !strings.Contains(out, "not permitted") { + t.Errorf("runc exec chroot must be denied: %v %s", err, out) + } +} From ec174141d7e11bf22c6106e6492da28b0e340115 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:49:13 +0200 Subject: [PATCH 23/34] fix(enforce_privilege): handle the bpf and perf_event_open signatures of Linux 6.15 Linux 6.15 added `bool kernel` to the bpf hook and dropped the attr argument of perf_event_open, so the previous-verdict check refused every 6.15+ kernel (and the old perf_event_open program would not load there). Both programs now read `ret` at the position of the running kernel: a CO-RE type match against security_list_options picks the branch at load. The BTF check returns a result per hook (LSMHookArgMismatches) and accepts both signatures for these two hooks. The other 21 hooks have the same argument count from 6.1 to 6.18 (checked in lsm_hook_defs.h). Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- .../gadgets/common/enforcement/lsmhooks.go | 59 +++++++++++++++---- .../common/enforcement/lsmhooks_test.go | 39 ++++++++++-- .../gadgets/enforce_privilege/program.bpf.c | 37 +++++++++--- .../enforce_privilege/runtime/runtime.go | 25 ++++---- .../enforce_privilege/runtime/runtime_test.go | 12 +++- 5 files changed, 137 insertions(+), 35 deletions(-) diff --git a/pkg/ebpf/gadgets/common/enforcement/lsmhooks.go b/pkg/ebpf/gadgets/common/enforcement/lsmhooks.go index 5bdec1263a..977f80b15d 100644 --- a/pkg/ebpf/gadgets/common/enforcement/lsmhooks.go +++ b/pkg/ebpf/gadgets/common/enforcement/lsmhooks.go @@ -4,26 +4,63 @@ package enforcement import ( + "errors" "fmt" + "sort" "github.com/cilium/ebpf/btf" ) -// CheckLSMHookArgs checks the argument count of each BPF LSM hook (name -> -// count) in the kernel's BTF. A BPF LSM program reads the previous BPF verdict, -// `ret`, right after the hook arguments, so a hook with another argument count -// would make it read an argument as the verdict. Call it before the programs -// are loaded, and refuse the kernel on an error. -func CheckLSMHookArgs(kernel *btf.Spec, hooks map[string]int) error { - for hook, args := range hooks { +// LSMHookArgMismatches checks the argument count of each BPF LSM hook in the +// kernel's BTF against the counts its program can handle (hook -> accepted +// counts). It returns one error per hook that does not match, so a caller can +// decide per hook. A BPF LSM program reads the previous BPF verdict, `ret`, +// right after the hook arguments, so a hook with another argument count would +// make it read an argument as the verdict. +func LSMHookArgMismatches(kernel *btf.Spec, hooks map[string][]int) map[string]error { + bad := map[string]error{} + for hook, accepted := range hooks { var fn *btf.Func if err := kernel.TypeByName("bpf_lsm_"+hook, &fn); err != nil { - return fmt.Errorf("LSM hook %s: %w", hook, err) + bad[hook] = fmt.Errorf("LSM hook %s: %w", hook, err) + continue } proto, ok := fn.Type.(*btf.FuncProto) - if !ok || len(proto.Params) != args { - return fmt.Errorf("LSM hook %s has an unexpected signature; this kernel is not supported", hook) + if !ok { + bad[hook] = fmt.Errorf("LSM hook %s has no function prototype", hook) + continue + } + known := false + for _, n := range accepted { + known = known || len(proto.Params) == n + } + if !known { + bad[hook] = fmt.Errorf("LSM hook %s has %d arguments, the program handles %v", hook, len(proto.Params), accepted) } } - return nil + return bad +} + +// CheckLSMHookArgs is LSMHookArgMismatches for programs with one signature +// per hook: it fails when any hook does not match. +func CheckLSMHookArgs(kernel *btf.Spec, hooks map[string]int) error { + accepted := make(map[string][]int, len(hooks)) + for hook, n := range hooks { + accepted[hook] = []int{n} + } + return JoinHookMismatches(LSMHookArgMismatches(kernel, accepted)) +} + +// JoinHookMismatches joins the errors of LSMHookArgMismatches in hook order. +func JoinHookMismatches(bad map[string]error) error { + hooks := make([]string, 0, len(bad)) + for hook := range bad { + hooks = append(hooks, hook) + } + sort.Strings(hooks) + errs := make([]error, 0, len(hooks)) + for _, hook := range hooks { + errs = append(errs, bad[hook]) + } + return errors.Join(errs...) } diff --git a/pkg/ebpf/gadgets/common/enforcement/lsmhooks_test.go b/pkg/ebpf/gadgets/common/enforcement/lsmhooks_test.go index e2f56914d0..d6ca52ae02 100644 --- a/pkg/ebpf/gadgets/common/enforcement/lsmhooks_test.go +++ b/pkg/ebpf/gadgets/common/enforcement/lsmhooks_test.go @@ -10,15 +10,23 @@ import ( "github.com/cilium/ebpf/btf" ) -func TestCheckLSMHookArgs(t *testing.T) { +// kernelWith builds a synthetic kernel BTF with bpf_lsm_ taking n +// pointer arguments. +func kernelWith(t *testing.T, hooks map[string]int) *btf.Spec { + t.Helper() b, err := btf.NewBuilder(nil, nil) if err != nil { t.Fatal(err) } ptr := &btf.Pointer{Target: &btf.Void{}} - proto := &btf.FuncProto{Return: &btf.Int{Name: "int", Size: 4, Encoding: btf.Signed}, Params: []btf.FuncParam{{Name: "a", Type: ptr}, {Name: "b", Type: ptr}}} - if _, err := b.Add(&btf.Func{Name: "bpf_lsm_two", Type: proto, Linkage: btf.GlobalFunc}); err != nil { - t.Fatal(err) + for hook, n := range hooks { + proto := &btf.FuncProto{Return: &btf.Int{Name: "int", Size: 4, Encoding: btf.Signed}} + for i := 0; i < n; i++ { + proto.Params = append(proto.Params, btf.FuncParam{Name: "a", Type: ptr}) + } + if _, err := b.Add(&btf.Func{Name: "bpf_lsm_" + hook, Type: proto, Linkage: btf.GlobalFunc}); err != nil { + t.Fatal(err) + } } raw, err := b.Marshal(nil, nil) if err != nil { @@ -28,6 +36,29 @@ func TestCheckLSMHookArgs(t *testing.T) { if err != nil { t.Fatal(err) } + return kernel +} + +func TestLSMHookArgMismatches(t *testing.T) { + // Before and from Linux 6.15: bpf has 3 then 4 arguments, perf_event_open 2 then 1. + accepted := map[string][]int{"bpf": {3, 4}, "perf_event_open": {2, 1}, "path_chroot": {1}} + for name, kernel := range map[string]map[string]int{ + "6.14": {"bpf": 3, "perf_event_open": 2, "path_chroot": 1}, + "6.15": {"bpf": 4, "perf_event_open": 1, "path_chroot": 1}, + } { + if bad := LSMHookArgMismatches(kernelWith(t, kernel), accepted); len(bad) != 0 { + t.Errorf("%s: %v", name, bad) + } + } + // One unknown signature and one missing hook: only those two are reported. + bad := LSMHookArgMismatches(kernelWith(t, map[string]int{"bpf": 5, "perf_event_open": 2}), accepted) + if len(bad) != 2 || bad["bpf"] == nil || bad["path_chroot"] == nil || bad["perf_event_open"] != nil { + t.Fatalf("per-hook result: %v", bad) + } +} + +func TestCheckLSMHookArgs(t *testing.T) { + kernel := kernelWith(t, map[string]int{"two": 2}) if err := CheckLSMHookArgs(kernel, map[string]int{"two": 2}); err != nil { t.Fatalf("matching count: %v", err) } diff --git a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c index 6ceca3b5b8..590b4fdd61 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c +++ b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c @@ -17,7 +17,8 @@ // other enforcement gadgets do. `ret` follows the hook arguments, so its // position depends on the hook signature: the operator checks every // argument count in the kernel's BTF (runtime.Hooks) and refuses a kernel -// where one differs. +// where one is unknown. bpf and perf_event_open have two known signatures +// (before and from 6.15) and pick theirs at load. // - Each hook body is a plain function, so it can also be attached as // fmod_ret on the matching security_* function. @@ -916,19 +917,41 @@ int BPF_PROG(ig_priv_module_request, char *kmod_name, int ret) /* bpf, perf, keys, mknod */ +// Linux 6.15 changed two hook signatures: bpf gained `bool kernel`, and +// perf_event_open lost its attr argument. `ret` follows the hook arguments, so +// these two programs read it at the position of the running kernel. The type +// match against the kernel's security_list_options is resolved at load (CO-RE), +// so the verifier sees only the branch that applies. +union bpf_attr___priv; +struct perf_event_attr___priv; +union security_list_options___bpf3 { + int (*bpf)(int, union bpf_attr___priv *, unsigned int); +}; +union security_list_options___perf2 { + int (*perf_event_open)(struct perf_event_attr___priv *, int); +}; + SEC("lsm/bpf") -int BPF_PROG(ig_priv_bpf, int cmd, union bpf_attr *attr, unsigned int size, int ret) +int BPF_PROG(ig_priv_bpf) { - if (ret) - return ret; + if (bpf_core_type_matches(union security_list_options___bpf3)) { + if ((int)ctx[3]) + return (int)ctx[3]; + } else if ((int)ctx[4]) { + return (int)ctx[4]; + } return check_syscall(ctx, PRIV_HOOK_BPF, PRIV_SC_BPF, 0); } SEC("lsm/perf_event_open") -int BPF_PROG(ig_priv_perf_event_open, struct perf_event_attr *attr, int type, int ret) +int BPF_PROG(ig_priv_perf_event_open) { - if (ret) - return ret; + if (bpf_core_type_matches(union security_list_options___perf2)) { + if ((int)ctx[2]) + return (int)ctx[2]; + } else if ((int)ctx[1]) { + return (int)ctx[1]; + } return check_syscall(ctx, PRIV_HOOK_PERF_EVENT_OPEN, PRIV_SC_PERF_EVENT_OPEN, 0); } diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go index 72ea45e68e..10d15d5fe2 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go @@ -31,16 +31,17 @@ const ( type ContainerIdentity = enforcement.ContainerIdentity -// Hooks lists the int LSM hooks of ../program.bpf.c with their argument count -// (without `ret`). Each program reads the previous verdict, `ret`, right after -// them, so the counts must match the running kernel. -var Hooks = map[string]int{ - "userns_create": 1, "task_alloc": 2, "capable": 4, "capset": 5, - "task_fix_setuid": 3, "task_fix_setgid": 3, "sb_mount": 5, "sb_kern_mount": 1, - "sb_umount": 2, "move_mount": 2, "sb_pivotroot": 2, "path_chroot": 1, - "inode_init_security_anon": 3, "ptrace_access_check": 2, "ptrace_traceme": 1, - "kernel_read_file": 3, "kernel_load_data": 2, "kernel_module_request": 1, - "bpf": 3, "perf_event_open": 2, "key_alloc": 3, "key_permission": 3, "inode_mknod": 4, +// Hooks lists the int LSM hooks of ../program.bpf.c with the argument counts +// (without `ret`) their program handles. Each program reads the previous +// verdict, `ret`, right after the arguments. bpf and perf_event_open changed in +// Linux 6.15 and handle both signatures (checked for 6.1 to 6.18). +var Hooks = map[string][]int{ + "userns_create": {1}, "task_alloc": {2}, "capable": {4}, "capset": {5}, + "task_fix_setuid": {3}, "task_fix_setgid": {3}, "sb_mount": {5}, "sb_kern_mount": {1}, + "sb_umount": {2}, "move_mount": {2}, "sb_pivotroot": {2}, "path_chroot": {1}, + "inode_init_security_anon": {3}, "ptrace_access_check": {2}, "ptrace_traceme": {1}, + "kernel_read_file": {3}, "kernel_load_data": {2}, "kernel_module_request": {1}, + "bpf": {3, 4}, "perf_event_open": {2, 1}, "key_alloc": {3}, "key_permission": {3}, "inode_mknod": {4}, } // checkKernel refuses a kernel whose hook signatures differ from Hooks @@ -50,7 +51,9 @@ var checkKernel = func() error { if err != nil { return fmt.Errorf("kernel BTF: %w", err) } - return enforcement.CheckLSMHookArgs(kernel, Hooks) + // A hook with an unknown signature cannot be skipped before load, and its + // program would read a wrong argument: refuse the kernel and name the hooks. + return enforcement.JoinHookMismatches(enforcement.LSMHookArgMismatches(kernel, Hooks)) } // Config holds the compiled rules and the initial container selection. diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go index 9c3c5a63c5..bd923b502f 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go @@ -85,11 +85,19 @@ func TestHooksMatchProgram(t *testing.T) { continue } seen[m[1]] = true + if len(Hooks[m[1]]) > 1 { + // Two signatures: the program reads ctx itself (see the + // security_list_options type matches in the program). + if strings.TrimSpace(m[2]) != "" { + t.Errorf("hook %s has two signatures and must not name arguments", m[1]) + } + continue + } if !strings.HasSuffix(strings.TrimSpace(m[2]), "int ret") { t.Errorf("hook %s does not end with int ret", m[1]) } - if args := strings.Count(m[2], ",") - 1; Hooks[m[1]] != args { - t.Errorf("hook %s: program has %d arguments, Hooks has %d", m[1], args, Hooks[m[1]]) + if args := strings.Count(m[2], ",") - 1; len(Hooks[m[1]]) != 1 || Hooks[m[1]][0] != args { + t.Errorf("hook %s: program has %d arguments, Hooks has %v", m[1], args, Hooks[m[1]]) } } if len(seen) != len(Hooks) { From 8e0a16812126d48fbd18770751082320da0036c2 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:57:06 +0200 Subject: [PATCH 24/34] docs(enforce_privilege): a verdict is one kernel check, not one syscall setns into a namespace checks CAP_SYS_ADMIN twice (the namespace user namespace and the caller's). Audit lets the first check pass, so one audited setns gives two identical verdicts; enforce stops at the first. A per-syscall dedupe is not reliable in an LSM hook (there is no syscall instance id, and a time window would merge real calls), so the README documents it and a kernel test pins it down. The test fixture now records the fields of each verdict. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/enforce_privilege/README.md | 8 +++ .../runtime/runtime_kernel_test.go | 62 ++++++++++++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index 45ec570175..5c69587563 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -185,3 +185,11 @@ key, the event names the rule `default` with `is_default` false. Rule and origin names are in the owner's compiled policy (`RuleNames`, `Origins`). Reporting is limited to 100 events per CPU per second; enforcement is never throttled. +A verdict is one kernel check, not one syscall. A syscall that checks the same +capability twice gives two identical verdicts in audit mode, because audit lets +the first check pass: `setns` checks `CAP_SYS_ADMIN` against the namespace's +user namespace and against the caller's, so an audited `setns` gives two. In +enforce mode the first deny stops the syscall, so it gives one. The `counters` +count checks the same way. A consumer that needs one event per syscall can +merge identical verdicts of one task that arrive together. + diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go index 124314071e..5c5f4e839f 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go @@ -27,8 +27,10 @@ import ( "os" "os/exec" "path/filepath" + goruntime "runtime" "strconv" "strings" + "sync" "sync/atomic" "testing" "time" @@ -60,6 +62,8 @@ type kernelFixture struct { container ContainerIdentity bindings *enforcement.Bindings verdicts atomic.Int32 + mu sync.Mutex + events []string // hook, syscall, capability, mode and comm of each verdict } func newKernelFixture(t *testing.T) *kernelFixture { @@ -106,7 +110,20 @@ func (f *kernelFixture) start(t *testing.T, name string, compiled privpolicy.Com if f.bindings, err = GetBindings(gc); err != nil { return err } - return gc.GetDataSources()["verdict"].Subscribe(func(datasource.DataSource, datasource.Data) error { + ds := gc.GetDataSources()["verdict"] + fields := map[string]datasource.FieldAccessor{} + for _, name := range []string{"hook", "syscall", "capability", "mode", "comm"} { + fields[name] = ds.GetField(name) + } + return ds.Subscribe(func(_ datasource.DataSource, data datasource.Data) error { + hook, _ := fields["hook"].String(data) + syscall, _ := fields["syscall"].String(data) + capability, _ := fields["capability"].Int32(data) + mode, _ := fields["mode"].Uint32(data) + comm, _ := fields["comm"].String(data) + f.mu.Lock() + f.events = append(f.events, fmt.Sprintf("%s %s cap=%d mode=%d %s", hook, syscall, capability, mode, comm)) + f.mu.Unlock() f.verdicts.Add(1) return nil }, 100) @@ -500,3 +517,46 @@ func TestWrongRuntimeNamespaceStillJudgesWorkload(t *testing.T) { t.Errorf("runc exec chroot must be denied: %v %s", err, out) } } + +// setnsNr is the setns syscall number of the test's architecture. +func setnsNr() int { + if goruntime.GOARCH == "arm64" { + return 268 + } + return 308 // amd64 +} + +// TestSetnsVerdictsPerCheck pins down that a verdict is one kernel check: +// setns into a mount namespace checks CAP_SYS_ADMIN twice, so audit mode +// reports both checks and enforce mode stops at the first (README, "Maps and +// events"). +func TestSetnsVerdictsPerCheck(t *testing.T) { + for mode, want := range map[string]int32{"audit": 2, "enforce": 1} { + t.Run(mode, func(t *testing.T) { + f := newKernelFixture(t) + rule := privpolicy.Rule{Name: "no-new-mntns", Action: "deny", Target: privpolicy.Target{Syscall: []string{"setns"}, Flags: []string{"CLONE_NEWNS"}}} + doc := privpolicy.Document{Version: 1, Name: "setns-" + mode, Policy: privpolicy.Policy{Mode: mode, Privilege: privpolicy.Section{Default: "allow", Rules: []privpolicy.Rule{rule}}}} + compiled, err := privpolicy.CompileKernel(doc, 1) + if err != nil { + t.Fatal(err) + } + f.start(t, "privilege-setns-"+mode, compiled, nil) + if err := f.bindings.Bind(enforcement.Binding{Container: f.container}); err != nil { + t.Fatal(err) + } + defer f.bindings.Unbind(f.container) + // One setns(2) call. nsenter is not used: it retries once after + // EPERM, which gives two syscalls in enforce mode. + err = f.exec("perl", "-e", fmt.Sprintf(`open(my $f, "<", "/proc/self/ns/mnt") or die; exit(syscall(%d, fileno($f), 0x20000) == 0 ? 0 : 1)`, setnsNr())) + if (mode == "enforce") != (err != nil) { + t.Fatalf("setns in %s mode: %v", mode, err) + } + time.Sleep(500 * time.Millisecond) + if got := f.verdicts.Load(); got != want { + f.mu.Lock() + defer f.mu.Unlock() + t.Fatalf("%s: %d verdicts, want %d: %v", mode, got, want, f.events) + } + }) + } +} From 9a53ab354c7967ffa6a0961ff472bfe8b1b5350e Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:06:57 +0200 Subject: [PATCH 25/34] fix(enforcement): arming converges while a container keeps forking The walk repeated until a pass saw no new task id, so a container that keeps starting short-lived tasks could fail the bind with "tasks still changing". A pass now counts only the tasks it had to arm: a task that already carries the mark (a child forked after its parent was armed) or that exited is not progress. The rescan for children forked before their parent was armed stays. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- .../gadgets/common/enforcement/arm_linux.go | 48 ++++++++++++++----- .../common/enforcement/arm_linux_test.go | 45 +++++++++++++++++ 2 files changed, 80 insertions(+), 13 deletions(-) diff --git a/pkg/ebpf/gadgets/common/enforcement/arm_linux.go b/pkg/ebpf/gadgets/common/enforcement/arm_linux.go index ce32d3a1f2..c6991e8a6a 100644 --- a/pkg/ebpf/gadgets/common/enforcement/arm_linux.go +++ b/pkg/ebpf/gadgets/common/enforcement/arm_linux.go @@ -33,8 +33,10 @@ const pidfdThread = unix.O_EXCL // // A task can fork between the read of cgroup.threads and its own arming; its // child is then not armed by inheritance. So the walk repeats until a pass -// arms no new task. It returns ErrContainerGone when the cgroup no longer -// exists. +// arms no task. A task that already carries the mark (a child forked after +// its parent was armed) or that exited is not progress, so a container that +// keeps starting short-lived tasks still converges. It returns +// ErrContainerGone when the cgroup no longer exists. func ArmExisting(armed Map, cgroupPath string) error { hostMntns, err := mntnsOf("1") if err != nil { @@ -43,7 +45,7 @@ func ArmExisting(armed Map, cgroupPath string) error { root := filepath.Join(host.HostRoot, "sys/fs/cgroup", cgroupPath) seen := map[int]bool{} for pass := 0; pass < maxArmingPasses; pass++ { - before := len(seen) + armedNow := 0 err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { if path == root { @@ -60,12 +62,14 @@ func ArmExisting(armed Map, cgroupPath string) error { if !d.IsDir() { return nil } - return armThreads(armed, filepath.Join(path, "cgroup.threads"), hostMntns, seen) + n, err := armThreads(armed, filepath.Join(path, "cgroup.threads"), hostMntns, seen) + armedNow += n + return err }) if err != nil { return err } - if len(seen) == before { + if armedNow == 0 { return nil } } @@ -77,15 +81,24 @@ var ErrContainerGone = errors.New("container cgroup is gone") const maxArmingPasses = 8 -func armThreads(armed Map, threads string, hostMntns uint64, seen map[int]bool) error { +// markLookup is the optional part of an arming map that reads a task's mark. +type markLookup interface { + Lookup(key, valueOut interface{}) error +} + +// armThreads arms the tasks of one cgroup.threads file and returns how many it +// had to arm. +func armThreads(armed Map, threads string, hostMntns uint64, seen map[int]bool) (int, error) { f, err := os.Open(threads) if errors.Is(err, fs.ErrNotExist) { - return nil // the cgroup was removed + return 0, nil // the cgroup was removed } if err != nil { - return fmt.Errorf("listing container tasks: %w", err) + return 0, fmt.Errorf("listing container tasks: %w", err) } defer f.Close() + lookup, canLookup := armed.(markLookup) + count := 0 scanner := bufio.NewScanner(f) for scanner.Scan() { tid, err := strconv.Atoi(scanner.Text()) @@ -100,7 +113,7 @@ func armThreads(armed Map, threads string, hostMntns uint64, seen map[int]bool) continue } if err != nil { - return fmt.Errorf("reading mount namespace of task %d: %w", tid, err) + return count, fmt.Errorf("reading mount namespace of task %d: %w", tid, err) } fd, err := unix.PidfdOpen(tid, pidfdThread) if errors.Is(err, unix.EINVAL) { @@ -116,22 +129,31 @@ func armThreads(armed Map, threads string, hostMntns uint64, seen map[int]bool) // Exited, unless the thread still runs: then it cannot be // armed, and the caller must not report it protected. if _, serr := os.Stat(filepath.Join(host.HostProcFs, scanner.Text())); serr == nil { - return fmt.Errorf("task %d runs but cannot be armed: %w", tid, err) + return count, fmt.Errorf("task %d runs but cannot be armed: %w", tid, err) } seen[tid] = true continue } if err != nil { - return fmt.Errorf("opening task %d: %w", tid, err) + return count, fmt.Errorf("opening task %d: %w", tid, err) + } + var mark uint32 + if canLookup && lookup.Lookup(uint32(fd), &mark) == nil && mark != 0 { + unix.Close(fd) // already armed, for example by inheritance + seen[tid] = true + continue } err = armed.Put(uint32(fd), uint32(1)) unix.Close(fd) if err != nil && !errors.Is(err, unix.ESRCH) { - return fmt.Errorf("arming task %d: %w", tid, err) + return count, fmt.Errorf("arming task %d: %w", tid, err) + } + if err == nil { + count++ } seen[tid] = true } - return scanner.Err() + return count, scanner.Err() } // openLeader opens a pidfd of the thread group leader of tid. All threads of a diff --git a/pkg/ebpf/gadgets/common/enforcement/arm_linux_test.go b/pkg/ebpf/gadgets/common/enforcement/arm_linux_test.go index 0b23c91d55..60252d1e1b 100644 --- a/pkg/ebpf/gadgets/common/enforcement/arm_linux_test.go +++ b/pkg/ebpf/gadgets/common/enforcement/arm_linux_test.go @@ -4,8 +4,11 @@ package enforcement import ( + "github.com/cilium/ebpf" "os" + "path/filepath" "strconv" + "strings" "testing" "golang.org/x/sys/unix" @@ -35,3 +38,45 @@ func TestOpenLeaderOfThread(t *testing.T) { t.Fatal("a task that does not exist has no leader") } } + +// markMap is an arming map whose Lookup reports every task as already armed, +// or none of them. +type markMap struct { + fakeMap + armedAll bool + puts int +} + +func (m *markMap) Put(interface{}, interface{}) error { m.puts++; return nil } +func (m *markMap) Lookup(_ interface{}, v interface{}) error { + if m.armedAll { + *v.(*uint32) = 1 + return nil + } + return ebpf.ErrKeyNotExist +} + +// Tasks that already carry the mark (children forked after their parent was +// armed) are not progress, so a container that keeps forking converges. +func TestArmThreadsCountsOnlyNewMarks(t *testing.T) { + entries, err := os.ReadDir("/proc/self/task") + if err != nil { + t.Skip("no /proc") + } + var tids []string + for _, e := range entries { + tids = append(tids, e.Name()) + } + threads := filepath.Join(t.TempDir(), "cgroup.threads") + if err := os.WriteFile(threads, []byte(strings.Join(tids, "\n")+"\n"), 0o600); err != nil { + t.Fatal(err) + } + armed := &markMap{armedAll: true} + if n, err := armThreads(armed, threads, 0, map[int]bool{}); err != nil || n != 0 || armed.puts != 0 { + t.Fatalf("already armed: n=%d puts=%d %v", n, armed.puts, err) + } + fresh := &markMap{} + if n, err := armThreads(fresh, threads, 0, map[int]bool{}); err != nil || n == 0 || n != fresh.puts { + t.Fatalf("unarmed: n=%d puts=%d %v", n, fresh.puts, err) + } +} From b4fbf875a8300951fc5032e2cf14c345c4df1c93 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Sun, 8 Nov 2026 11:33:42 +0200 Subject: [PATCH 26/34] feat(enforce_privilege): keep an explicit empty origin apart from no origin Rule origins and the new section default origin are pointers in the policy document. The compiler gives an explicit "" its own origin id; id 0 still means no origin. Compiled.DefaultOrigins lists, per slot, the default origins of the documents that agree with the merged default, for events of the default. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/enforce_privilege/README.md | 5 ++- .../enforce_privilege/policy/kernel.go | 37 +++++++++++---- .../enforce_privilege/policy/kernel_test.go | 45 +++++++++++++++++++ .../enforce_privilege/policy/policy.go | 8 +++- 4 files changed, 84 insertions(+), 11 deletions(-) diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index 5c69587563..e7b28e3bb4 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -182,7 +182,10 @@ The `verdict` datasource shares `generation`, `cgroup`, `deny`, `mode` and `comm`, `hook`, `syscall`, `flags`, `capability`, `fstype` and `slot`. When a document's default deny strengthens another document's audit rule on the same key, the event names the rule `default` with `is_default` false. Rule and origin -names are in the owner's compiled policy (`RuleNames`, `Origins`). Reporting is +names are in the owner's compiled policy (`RuleNames`, `Origins`). Origin id 0 +means no origin; an explicit empty origin has its own id. A default verdict +(rule `default`) takes its origins from `DefaultOrigins[slot]`: the default +origins of the documents that agree with the slot's merged default. Reporting is limited to 100 events per CPU per second; enforcement is never throttled. A verdict is one kernel check, not one syscall. A syscall that checks the same diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go index 2cda0ae362..592a825506 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go @@ -92,7 +92,8 @@ type KernelEntry struct { } // Compiled is the frozen map content plus the id tables events refer to. -// RuleNames[0] and Origins[0] are empty: id 0 means none. +// RuleNames[0] is empty and Origins[0] means "no origin": id 0 is none. An +// explicit empty origin has its own id, whose entry is "". type Compiled struct { Generation uint64 // Slots is the number of policy slots (combinations of policies). @@ -100,6 +101,11 @@ type Compiled struct { Entries []KernelEntry RuleNames []string Origins []string + // DefaultOrigins holds, per slot, the sorted distinct origins of the + // documents whose default equals the slot's merged default (a document + // without a default origin adds nothing). Events of the default (rule + // "default") take their origins from here. + DefaultOrigins [][]string } func rank(action, mode string) uint8 { @@ -140,16 +146,16 @@ func CompileSlots(slots [][]Document, generation uint64) (Compiled, error) { } c := Compiled{Generation: generation, RuleNames: []string{"", "default"}, Origins: []string{""}} originIDs := map[string]uint32{} - originID := func(o string) uint32 { - if o == "" { + originID := func(o *string) uint32 { + if o == nil { return 0 } - if id, ok := originIDs[o]; ok { + if id, ok := originIDs[*o]; ok { return id } - c.Origins = append(c.Origins, o) - originIDs[o] = uint32(len(c.Origins) - 1) - return originIDs[o] + c.Origins = append(c.Origins, *o) + originIDs[*o] = uint32(len(c.Origins) - 1) + return originIDs[*o] } for slot, docs := range slots { if len(docs) == 0 { @@ -171,7 +177,7 @@ func CompileSlots(slots [][]Document, generation uint64) (Compiled, error) { return c, nil } -func compileSlot(slot uint32, docs []Document, generation uint64, c *Compiled, originID func(string) uint32) ([]KernelEntry, error) { +func compileSlot(slot uint32, docs []Document, generation uint64, c *Compiled, originID func(*string) uint32) ([]KernelEntry, error) { meta := KernelEntry{Key: KernelKey{Slot: slot, Kind: KindMeta}, Value: KernelValue{Generation: generation}} var active []Document for _, d := range docs { @@ -186,6 +192,7 @@ func compileSlot(slot uint32, docs []Document, generation uint64, c *Compiled, o } } if len(active) == 0 { + c.DefaultOrigins = append(c.DefaultOrigins, nil) return []KernelEntry{meta}, nil } @@ -226,6 +233,20 @@ func compileSlot(slot uint32, docs []Document, generation uint64, c *Compiled, o } } def := KernelValue{Rank: rank(merged, mode), RuleID: 1} + // The documents that agree with the merged default contribute their + // default origin; an explicit "" is kept, a missing one adds nothing. + origins := map[string]bool{} + for _, d := range active { + if d.Policy.Privilege.Default == merged && d.Policy.Privilege.DefaultOrigin != nil { + origins[*d.Policy.Privilege.DefaultOrigin] = true + } + } + var slotOrigins []string + for o := range origins { + slotOrigins = append(slotOrigins, o) + } + sort.Strings(slotOrigins) + c.DefaultOrigins = append(c.DefaultOrigins, slotOrigins) put(KernelKey{Kind: KindDefault, ID: KindSyscall}, def) put(KernelKey{Kind: KindDefault, ID: KindMountFSType}, def) diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go index d3766d5efb..a44db590ff 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go @@ -270,3 +270,48 @@ func TestParse(t *testing.T) { } } } + +// An absent origin and an explicit "" are different: absent has id 0 (none), +// "" gets its own id. Default origins are kept per slot the same way. +func TestOriginPresence(t *testing.T) { + empty, label := "", "catalog/P1" + doc := func(name, def string, defaultOrigin *string, rules ...Rule) Document { + return Document{Version: 1, Name: name, Policy: Policy{Mode: "enforce", Privilege: Section{Default: def, DefaultOrigin: defaultOrigin, Rules: rules}}} + } + rule := func(name, syscall string, origin *string) Rule { + return Rule{Name: name, Action: "deny", Origin: origin, Target: Target{Syscall: []string{syscall}}} + } + c, err := CompileSlots([][]Document{ + {doc("a", "deny", &empty, rule("none", "mount", nil), rule("blank", "bpf", &empty), rule("set", "chroot", &label))}, + {doc("b", "deny", nil), doc("c", "deny", &label), doc("d", "allow", &empty)}, + {doc("e", "allow", nil)}, + }, 1) + if err != nil { + t.Fatal(err) + } + originOf := func(syscall string) (uint32, string) { + v, ok := lookup(c, KernelKey{Kind: KindSyscall, ID: Syscalls[syscall]}) + if !ok { + t.Fatalf("missing %s", syscall) + } + return v.OriginID, c.Origins[v.OriginID] + } + if id, _ := originOf("mount"); id != 0 { + t.Errorf("absent rule origin: id %d, want 0", id) + } + if id, o := originOf("bpf"); id == 0 || o != "" { + t.Errorf("explicit empty rule origin: id %d %q, want its own id", id, o) + } + if _, o := originOf("chroot"); o != label { + t.Errorf("rule origin %q", o) + } + want := [][]string{{""}, {label}, nil} + if len(c.DefaultOrigins) != len(want) { + t.Fatalf("default origins per slot: %q", c.DefaultOrigins) + } + for slot := range want { + if strings.Join(c.DefaultOrigins[slot], ",") != strings.Join(want[slot], ",") || (c.DefaultOrigins[slot] == nil) != (want[slot] == nil) { + t.Errorf("slot %d default origins %q, want %q", slot, c.DefaultOrigins[slot], want[slot]) + } + } +} diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go b/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go index 92b018dd3a..e7bbb868f2 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go @@ -34,7 +34,10 @@ type Policy struct { type Section struct { Default string `yaml:"default" json:"default"` - Rules []Rule `yaml:"rules,omitempty" json:"rules,omitempty"` + // DefaultOrigin labels the default. Absent (nil) and an explicit "" are + // different: only a present label is reported. + DefaultOrigin *string `yaml:"defaultOrigin,omitempty" json:"defaultOrigin,omitempty"` + Rules []Rule `yaml:"rules,omitempty" json:"rules,omitempty"` } type Rule struct { @@ -42,7 +45,8 @@ type Rule struct { Target Target `yaml:"target" json:"target"` Action string `yaml:"action" json:"action"` Mode string `yaml:"mode,omitempty" json:"mode,omitempty"` - Origin string `yaml:"origin,omitempty" json:"origin,omitempty"` + // Origin labels the rule; absent (nil) and an explicit "" are different. + Origin *string `yaml:"origin,omitempty" json:"origin,omitempty"` } // Target selects exactly one primitive kind. Flags qualify Syscall only. From c05ec597337c9074c71e186a9ea8bc28775a9184 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Sun, 8 Nov 2026 11:34:08 +0200 Subject: [PATCH 27/34] fix(enforce_privilege): the default follows the section's defaultMode A section can set the mode of its default apart from the policy mode. The compiler ignored it, so an audit policy with an enforce default only audited its default deny. The merged default and the audit-deny raise now use each document's defaultMode, else its policy mode. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- .../enforce_privilege/policy/kernel.go | 4 +-- .../enforce_privilege/policy/kernel_test.go | 27 +++++++++++++++++++ .../enforce_privilege/policy/policy.go | 14 ++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go index 592a825506..a27d66a394 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go @@ -228,7 +228,7 @@ func compileSlot(slot uint32, docs []Document, generation uint64, c *Compiled, o } mode := "audit" for _, d := range active { - if d.Policy.Privilege.Default == merged && d.Policy.Mode == "enforce" { + if d.Policy.Privilege.Default == merged && d.defaultMode() == "enforce" { mode = "enforce" } } @@ -297,7 +297,7 @@ func compileSlot(slot uint32, docs []Document, generation uint64, c *Compiled, o if d.Policy.Privilege.Default != "deny" || hasRule(i) { continue } - if r := rank("deny", d.Policy.Mode); r > v.Rank { + if r := rank("deny", d.defaultMode()); r > v.Rank { v = KernelValue{Rank: r, RuleID: 1} } } diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go index a44db590ff..0fe17c26d9 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go @@ -315,3 +315,30 @@ func TestOriginPresence(t *testing.T) { } } } + +// The default takes the section's defaultMode when set, else the policy mode. +func TestDefaultMode(t *testing.T) { + doc := func(mode, defaultMode string) Document { + return Document{Version: 1, Policy: Policy{Mode: mode, Privilege: Section{Default: "deny", DefaultMode: defaultMode}}} + } + for _, tc := range []struct { + mode, defaultMode string + want uint8 + }{ + {"enforce", "", RankDenyEnforce}, + {"enforce", "audit", RankDenyAudit}, + {"audit", "enforce", RankDenyEnforce}, + {"audit", "", RankDenyAudit}, + } { + c, err := CompileKernel(doc(tc.mode, tc.defaultMode), 1) + if err != nil { + t.Fatal(err) + } + if v, _ := lookup(c, KernelKey{Kind: KindDefault, ID: KindSyscall}); v.Rank != tc.want { + t.Errorf("mode %s defaultMode %q: rank %d, want %d", tc.mode, tc.defaultMode, v.Rank, tc.want) + } + } + if _, err := parse("version: 1\npolicy:\n mode: enforce\n privilege:\n default: deny\n defaultMode: block\n"); err == nil { + t.Error("an invalid defaultMode must be rejected") + } +} diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go b/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go index e7bbb868f2..78924ae9be 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go @@ -34,6 +34,9 @@ type Policy struct { type Section struct { Default string `yaml:"default" json:"default"` + // DefaultMode is the mode of the default (audit or enforce); empty takes + // the policy mode. + DefaultMode string `yaml:"defaultMode,omitempty" json:"defaultMode,omitempty"` // DefaultOrigin labels the default. Absent (nil) and an explicit "" are // different: only a present label is reported. DefaultOrigin *string `yaml:"defaultOrigin,omitempty" json:"defaultOrigin,omitempty"` @@ -85,6 +88,14 @@ func validMode(m string, allowEmpty bool) bool { func validAction(a string) bool { return a == "allow" || a == "deny" } +// defaultMode is the mode of the document's default. +func (d Document) defaultMode() string { + if d.Policy.Privilege.DefaultMode != "" { + return d.Policy.Privilege.DefaultMode + } + return d.Policy.Mode +} + func (d Document) Validate() error { if d.Version != 1 { return fmt.Errorf("unsupported policy version %d", d.Version) @@ -97,6 +108,9 @@ func (d Document) Validate() error { if !validAction(s.Default) { return fmt.Errorf("privilege default must be allow or deny") } + if !validMode(s.DefaultMode, true) { + return fmt.Errorf("privilege defaultMode must be audit or enforce") + } if p.Mode == "off" && len(s.Rules) > 0 { return fmt.Errorf("an off policy cannot carry rules") } From d2200c119f3a801bc88ab243076799e1bbb7f6bc Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:40:41 +0200 Subject: [PATCH 28/34] fix(enforcement): exact binding scope, fail closed on a missing slot, nonblocking Audit faults Answers the alignment review on the privilege gadget: - Scope is exact by default, as in enforce_exec. A binding covers its own cgroup; BINDING_SUBTREE extends one binding to its descendant cgroups. The lookup walks parents from the top down, so a subtree binding covers a task at any depth, and the controller refuses a subtree binding deeper than the walk. - The binding value carries the slot's class (BINDING_STRICT, from the slot's META mode). A bound cgroup whose slot has no META entry is a structural fault: a Strict binding denies and counts a slot fault, an Audit binding allows. Both count a section error. A test-only parameter (policy-fail-headers) injects the fault. - A child mark that fails outside a Strict binding no longer refuses the fork, also with CLONE_INTO_CGROUP. That case counts a node fault in the new arming_witness map, which the owner reports against every Strict binding. - The runtime helper exemption is an owner-admitted binding flag (BINDING_RUNTIME_HELPERS, needs BINDING_SUBTREE). Container bindings have none. Exempted execs are counted. Kernel tests: missing slot policy in both modes, hooks judged without the exemption, exact scope skips descendants, and task_alloc next to another BPF LSM program in both attach orders. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/common/README.md | 6 +- pkg/ebpf/gadgets/common/arming.h | 66 ++++- pkg/ebpf/gadgets/common/bindings.h | 55 +++- .../gadgets/common/enforcement/bindings.go | 118 ++++++++- .../common/enforcement/bindings_test.go | 90 ++++++- pkg/ebpf/gadgets/enforce_privilege/README.md | 64 +++-- .../gadgets/enforce_privilege/gadget.yaml | 4 + .../enforce_privilege/privilege_policy.h | 34 ++- .../gadgets/enforce_privilege/program.bpf.c | 82 +++--- .../enforce_privilege/runtime/runtime.go | 14 + .../runtime/runtime_kernel_test.go | 241 +++++++++++++++++- .../enforce_privilege/runtime/runtime_test.go | 15 ++ 12 files changed, 671 insertions(+), 118 deletions(-) diff --git a/pkg/ebpf/gadgets/common/README.md b/pkg/ebpf/gadgets/common/README.md index 6e50274597..7fe0d2f235 100644 --- a/pkg/ebpf/gadgets/common/README.md +++ b/pkg/ebpf/gadgets/common/README.md @@ -14,12 +14,12 @@ and keeps its path rules in an array that `policy.h` does not describe. | File | Content | |---|---| -| `bindings.h` | `bindings` map: cgroup id → the container's policy slot; optional parent-cgroup depth | +| `bindings.h` | `bindings` map: cgroup id → the container's policy slot and flags. Exact scope by default; `BINDING_SUBTREE` covers descendant cgroups, `BINDING_STRICT` carries the slot's class (a fault denies), `BINDING_RUNTIME_HELPERS` admits the arming exemption | | `policy.h` | policy modes (0 off, 1 audit, 2 enforce), rank resolution (explicit deny > allow > default, one mode per rule), shared policy value | | `verdict.h` | first verdict fields (`generation`, `cgroup`, `deny`, `mode`), per-CPU `counters` (denied, submitted, dropped, allowed, section error, rate limited), 100 reports per CPU per second | -| `arming.h` | arming rule: judge only tasks that executed outside the runtime mount namespace (not those forked there by an unarmed runtime helper, such as OCI hooks), and their children; `arming_faults`, tracking faults per slot | +| `arming.h` | arming rule: judge only tasks that executed outside the runtime mount namespace, and their children; under `BINDING_RUNTIME_HELPERS` only, not those forked there by an unarmed runtime helper (OCI hooks). `arming_faults`: faults per slot; `arming_witness`: node faults and exempted execs. Include `bindings.h` first | | `merge/` | the merge contract for several policies on one container, and its shared test vectors | -| `enforcement/` | Go: the binding controller of one node-wide instance (policy written and frozen before any binding, live `Bind`/`Unbind` of a container to a slot, readiness, arming of tasks already running, `SlotFaults`) | +| `enforcement/` | Go: the binding controller of one node-wide instance (policy written and frozen before any binding, live `Bind`/`Unbind` of a container to a slot, readiness, arming of tasks already running, `SlotFaults`, `ArmingWitness`) | Section-specific parts (the policy key layout, hooks, compiler) stay in each gadget. diff --git a/pkg/ebpf/gadgets/common/arming.h b/pkg/ebpf/gadgets/common/arming.h index b7bf0f222d..9ee13ded87 100644 --- a/pkg/ebpf/gadgets/common/arming.h +++ b/pkg/ebpf/gadgets/common/arming.h @@ -20,12 +20,16 @@ // // OCI createContainer and startContainer hooks are runtime setup too: runc init // forks them inside the container mount namespace and cgroup, before the -// workload starts. So a task is not armed at exec when its parent is unarmed, -// outside the host mount namespace, and in the same mount namespace and cgroup -// as the task (a runtime helper inside the container). The workload and an -// "exec" command are execed by runc init itself, whose parent (the runtime or -// its shim) is in the host mount namespace. Such a helper, and a hook, are -// trusted not to leave a process behind: its children stay unarmed. +// workload starts. Only a binding with BINDING_RUNTIME_HELPERS exempts them: +// the owner sets it on a parent cgroup bound before its containers start, never +// on a container binding. Under that flag a task is not armed at exec when its +// parent is unarmed, outside the host mount namespace, and in the same mount +// namespace and cgroup as the task (a runtime helper inside the container). +// The workload and an "exec" command are execed by runc init itself, whose +// parent (the runtime or its shim) is in the host mount namespace. Such a +// helper, and a hook, are trusted not to leave a process behind: its children +// stay unarmed. Every exempted exec is counted. A Strict container binding has +// no exemption, as in enforce_exec. #ifndef __ENFORCEMENT_ARMING_H #define __ENFORCEMENT_ARMING_H @@ -34,6 +38,12 @@ #include #include +// The binding flags come from bindings.h, which a gadget includes first +// (after it sets BINDINGS_CGROUP_DEPTH). +#ifndef BINDING_RUNTIME_HELPERS +#error "include bindings.h before arming.h" +#endif + // Inode number of the mount namespace the container runtime runs in. 0 means // the mount namespace of the global init task. Set it when the runtime does not // run in init's mount namespace (for example a kind node). @@ -66,10 +76,10 @@ static __always_inline bool arming_task_armed(struct task_struct *task) // set it in production. const volatile bool arming_fail_fork_marks = false; -// Tracking faults per policy slot, for the owner's health check: a mark that -// failed for a task in a bound cgroup whose slot enforces. The array is -// preallocated, so counting never fails. A new instance starts at zero and -// arms the running tasks again. +// Faults per policy slot, for the owner's health check: a mark that failed for +// a task in a Strict binding, or a Strict binding whose slot has no policy. The +// array is preallocated, so counting never fails. A new instance starts at zero +// and arms the running tasks again. #ifndef ARMING_FAULT_SLOTS #define ARMING_FAULT_SLOTS 64 #endif @@ -89,6 +99,32 @@ static __always_inline void arming_slot_fault(__u32 slot) __sync_fetch_and_add(value, 1); } +// Node-wide witness counters of the arming rule, for the owner's health check. +enum arming_witness { + // A child mark failed outside a Strict binding. The child can be in + // another cgroup (clone3 with CLONE_INTO_CGROUP), so the owner treats a + // nonzero count as a fault of every Strict binding. + ARMING_WITNESS_NODE_FAULT = 0, + // Execs that BINDING_RUNTIME_HELPERS left unarmed. + ARMING_WITNESS_EXEMPT_EXEC = 1, + ARMING_WITNESS_MAX = 2, +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, ARMING_WITNESS_MAX); + __type(key, __u32); + __type(value, __u64); +} arming_witness SEC(".maps"); + +static __always_inline void arming_witness_count(__u32 key) +{ + __u64 *value = bpf_map_lookup_elem(&arming_witness, &key); + + if (value) + __sync_fetch_and_add(value, 1); +} + // arming_mark returns false when the task storage could not be allocated (memory // pressure): the task then stays unarmed, and the caller handles the failure. static __always_inline bool arming_mark(struct task_struct *task) @@ -156,9 +192,10 @@ static __always_inline bool arming_runtime_child(struct task_struct *task, BPF_CORE_READ(task, cgroups, dfl_cgrp, kn, id); } -// arming_on_exec runs after a successful exec (bprm_committed_creds). +// arming_on_exec runs after a successful exec (bprm_committed_creds), with the +// flags of the binding that covers the task (0 when unbound). // arming_on_exec returns false when a needed mark failed. -static __always_inline bool arming_on_exec(void) +static __always_inline bool arming_on_exec(__u32 binding_flags) { struct task_struct *task = bpf_get_current_task_btf(); __u64 mntns = BPF_CORE_READ(task, nsproxy, mnt_ns, ns.inum); @@ -170,8 +207,11 @@ static __always_inline bool arming_on_exec(void) if (mntns == host) return true; // An unknown host namespace trusts no parent. - if (host && arming_runtime_child(task, mntns, host)) + if ((binding_flags & BINDING_RUNTIME_HELPERS) && host && + arming_runtime_child(task, mntns, host)) { + arming_witness_count(ARMING_WITNESS_EXEMPT_EXEC); return true; + } return arming_mark(task); } diff --git a/pkg/ebpf/gadgets/common/bindings.h b/pkg/ebpf/gadgets/common/bindings.h index c3101978e3..a85b55bbee 100644 --- a/pkg/ebpf/gadgets/common/bindings.h +++ b/pkg/ebpf/gadgets/common/bindings.h @@ -3,10 +3,16 @@ // Container binding map of a node-wide enforcement gadget instance: // key: cgroup v2 id -// value: the policy slot of the container (one combination of policies) +// value: the policy slot of the container (one combination of policies) and +// the binding flags // Same name, key and value size as the enforce_exec `bindings` map of the // Inspektor Gadget fork. The value meaning differs: there it is 0 or a runtime // mount namespace inode (restricted binding). +// +// Scope is exact by default, as in enforce_exec: a binding covers the tasks of +// its own cgroup only. BINDING_SUBTREE extends one binding to the descendant +// cgroups. The owner sets it only for a parent cgroup bound before its containers +// start, and reports it in status. #ifndef __ENFORCEMENT_BINDINGS_H #define __ENFORCEMENT_BINDINGS_H @@ -17,15 +23,23 @@ #define BINDINGS_MAX 1024 -// Number of cgroup levels checked: the task's cgroup and its parents. 1 (the -// enforce_exec behaviour) checks the task's own cgroup only. +// Number of cgroup levels checked for a BINDING_SUBTREE parent. 1 (the +// enforce_exec behaviour) checks the task's own cgroup only. A subtree binding +// covers its descendants at any depth when its own level is below this value; +// the owner refuses a subtree binding at a deeper level. #ifndef BINDINGS_CGROUP_DEPTH #define BINDINGS_CGROUP_DEPTH 1 #endif +// binding_value.flags +#define BINDING_SUBTREE (1U << 0) // also covers the descendant cgroups +#define BINDING_STRICT (1U << 1) // the slot enforces: a fault denies +// Owner-admitted runtime helper exemption (see arming_on_exec in arming.h). +#define BINDING_RUNTIME_HELPERS (1U << 2) + struct binding_value { __u32 slot; - __u32 reserved; + __u32 flags; }; struct { @@ -35,28 +49,41 @@ struct { __type(value, struct binding_value); } bindings SEC(".maps"); -// binding_lookup returns true when the current task's cgroup (or a parent up -// to BINDINGS_CGROUP_DEPTH) is bound, with the bound cgroup id and its slot. -static __always_inline bool binding_lookup(__u64 *cgroup, __u32 *slot) +// binding_lookup returns true when a binding covers the current task: the +// binding of the task's own cgroup, or else the nearest parent binding with +// BINDING_SUBTREE. It returns the bound cgroup id, the slot and the flags. +static __always_inline bool binding_lookup(__u64 *cgroup, __u32 *slot, + __u32 *flags) { - struct task_struct *task = bpf_get_current_task_btf(); __u64 id = bpf_get_current_cgroup_id(); struct binding_value *value = bpf_map_lookup_elem(&bindings, &id); #if BINDINGS_CGROUP_DEPTH > 1 - int level = BPF_CORE_READ(task, cgroups, dfl_cgrp, level); + if (!value) { + struct task_struct *task = bpf_get_current_task_btf(); + int level = BPF_CORE_READ(task, cgroups, dfl_cgrp, level); + struct binding_value *parent; + __u64 parent_id; - for (int i = 1; i < BINDINGS_CGROUP_DEPTH && !value; i++) { - if (level - i < 1) - break; - id = bpf_get_current_ancestor_cgroup_id(level - i); - value = bpf_map_lookup_elem(&bindings, &id); + // From the top down, so a subtree binding covers a task at any + // depth below it. The last match is the nearest parent. + for (int i = 1; i < BINDINGS_CGROUP_DEPTH; i++) { + if (i >= level) + break; + parent_id = bpf_get_current_ancestor_cgroup_id(i); + parent = bpf_map_lookup_elem(&bindings, &parent_id); + if (parent && (parent->flags & BINDING_SUBTREE)) { + value = parent; + id = parent_id; + } + } } #endif if (!value) return false; *cgroup = id; *slot = value->slot; + *flags = value->flags; return true; } diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings.go b/pkg/ebpf/gadgets/common/enforcement/bindings.go index e578bcaf68..4052dd2682 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings.go @@ -33,12 +33,31 @@ type ContainerIdentity = execruntime.ContainerIdentity type Binding struct { Container ContainerIdentity Slot uint32 + // Subtree also covers the descendant cgroups. Scope is exact by default; + // set it only for a parent cgroup bound before its containers start, and + // report it in status. + Subtree bool + // RuntimeHelpers admits the runtime helper exemption of ../arming.h for + // the tasks this binding covers (OCI hooks that runc init forks before the + // workload starts). Set it only with Subtree on such a parent cgroup. + RuntimeHelpers bool } +// Flags of struct binding_value in ../bindings.h. +const ( + bindingSubtree uint32 = 1 << 0 + bindingStrict uint32 = 1 << 1 + bindingRuntimeHelpers uint32 = 1 << 2 +) + +// MaxSubtreeLevel is the deepest cgroup level of a Subtree binding +// (BINDINGS_CGROUP_DEPTH - 1 in the gadgets that cover subtrees). +const MaxSubtreeLevel = 31 + // bindingValue matches struct binding_value in ../bindings.h. type bindingValue struct { - Slot uint32 - Reserved uint32 + Slot uint32 + Flags uint32 } // Map is the part of *ebpf.Map the controller uses. @@ -70,9 +89,16 @@ type Config struct { // ArmMap names the arming task-storage map. When set, binding a container // also arms the tasks already running in it (see ../arming.h). ArmMap string - // FaultMap names the per-slot tracking fault array (arming_faults in + // FaultMap names the per-slot fault array (arming_faults in // ../arming.h). When set, SlotFaults reads it. FaultMap string + // WitnessMap names the node-wide witness array (arming_witness in + // ../arming.h). When set, ArmingWitness reads it. + WitnessMap string + // StrictSlot reports whether a slot enforces. The class goes into each + // binding (BINDING_STRICT), so the program fails closed for a Strict + // binding even when the slot's policy is missing. Nil: no slot enforces. + StrictSlot func(slot uint32) bool } // FaultMap is the part of *ebpf.Map that SlotFaults reads. @@ -105,7 +131,7 @@ type Bindings struct { // not arm, so a retry with the same slot arms them. unarmed map[uint64]bool members, armed Map - faults FaultMap + faults, witness FaultMap ready, stopped bool startupComplete bool startupError error @@ -194,9 +220,36 @@ func (b *Bindings) validate(c Binding) error { if c.Slot >= b.config.Slots { return fmt.Errorf("policy slot %d out of range", c.Slot) } + if c.RuntimeHelpers && !c.Subtree { + return fmt.Errorf("the runtime helper exemption needs a subtree binding") + } + if c.Subtree && cgroupLevel(c.Container.CgroupPath) > MaxSubtreeLevel { + // The program would not see the binding from its descendants. + return fmt.Errorf("subtree binding deeper than cgroup level %d", MaxSubtreeLevel) + } return validate(c.Container) } +// cgroupLevel is the cgroup v2 level of a path ("/a/b" is 2). +func cgroupLevel(path string) int { + return strings.Count(strings.Trim(path, "/"), "/") + 1 +} + +// value is the kernel binding value of c. +func (b *Bindings) value(c Binding) bindingValue { + v := bindingValue{Slot: c.Slot} + if c.Subtree { + v.Flags |= bindingSubtree + } + if b.config.StrictSlot != nil && b.config.StrictSlot(c.Slot) { + v.Flags |= bindingStrict + } + if c.RuntimeHelpers { + v.Flags |= bindingRuntimeHelpers + } + return v +} + func validate(c ContainerIdentity) error { if strings.TrimSpace(c.Runtime) == "" || strings.TrimSpace(c.ID) == "" || c.CgroupID == 0 || c.CgroupPath == "/" || !strings.HasPrefix(c.CgroupPath, "/") { return fmt.Errorf("binding requires runtime, full container ID and non-root cgroup v2 identity") @@ -213,7 +266,7 @@ func (b *Bindings) bind(c Binding) error { if bound && previous.Container != id { return fmt.Errorf("cgroup identity conflicts with active binding") } - if bound && previous.Slot == c.Slot && !b.unarmed[id.CgroupID] { + if bound && b.value(previous) == b.value(c) && !b.unarmed[id.CgroupID] { return nil } if !bound { @@ -226,7 +279,7 @@ func (b *Bindings) bind(c Binding) error { return fmt.Errorf("container binding capacity exceeded") } } - if err := b.members.Put(id.CgroupID, bindingValue{Slot: c.Slot}); err != nil { + if err := b.members.Put(id.CgroupID, b.value(c)); err != nil { return fmt.Errorf("writing container binding: %w", err) } // A slot move arms again too: a task that the old slot left unarmed (an @@ -236,7 +289,7 @@ func (b *Bindings) bind(c Binding) error { if err := b.armExisting(b.armed, id.CgroupPath); err != nil { if bound && !errors.Is(err, ErrContainerGone) { // Undo the move, so a retry moves and arms again. - if perr := b.members.Put(id.CgroupID, bindingValue{Slot: previous.Slot}); perr != nil { + if perr := b.members.Put(id.CgroupID, b.value(previous)); perr != nil { b.active[id.CgroupID] = c b.unarmed[id.CgroupID] = true return errors.Join(err, perr) @@ -303,6 +356,15 @@ func (b *Bindings) Start(gc operators.GadgetContext) error { return fmt.Errorf("invalid fault map layout") } } + var witness FaultMap + if b.config.WitnessMap != "" { + if witness, err = getMap(b.config.WitnessMap); err != nil { + return err + } + if witness.Type() != ebpf.Array || witness.KeySize() != 4 || witness.ValueSize() != 8 || witness.MaxEntries() != armingWitnessCount { + return fmt.Errorf("invalid witness map layout") + } + } var armed Map if b.config.ArmMap != "" { if armed, err = getMap(b.config.ArmMap); err != nil { @@ -312,12 +374,42 @@ func (b *Bindings) Start(gc operators.GadgetContext) error { return fmt.Errorf("invalid arming map layout") } } - return b.start(policy, members, armed, faults) + return b.start(policy, members, armed, faults, witness) +} + +// armingWitnessCount is ARMING_WITNESS_MAX in ../arming.h. +const armingWitnessCount = 2 + +// ArmingWitness holds the node-wide witness counters of ../arming.h since the +// instance started. +type ArmingWitness struct { + // NodeFaults: a child mark failed outside a Strict binding, for a child + // that may start in any cgroup. A nonzero count is a fault of every + // Strict binding. + NodeFaults uint64 + // ExemptExecs: execs that the runtime helper exemption left unarmed. + ExemptExecs uint64 +} + +// ArmingWitness reads the witness counters. +func (b *Bindings) ArmingWitness() (ArmingWitness, error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.witness == nil { + return ArmingWitness{}, fmt.Errorf("no witness map configured") + } + var w [armingWitnessCount]uint64 + for i := range w { + if err := b.witness.Lookup(uint32(i), &w[i]); err != nil { + return ArmingWitness{}, fmt.Errorf("reading witness counter %d: %w", i, err) + } + } + return ArmingWitness{NodeFaults: w[0], ExemptExecs: w[1]}, nil } -// SlotFaults returns the tracking faults counted for a slot since the instance -// started (see ../arming.h). A nonzero count means a task of an enforcing slot -// may be outside the rules, or was refused a fork. +// SlotFaults returns the faults counted for a slot since the instance started +// (see ../arming.h). A nonzero count means a task of a Strict binding may be +// outside the rules, was refused a fork, or met a slot with no policy. func (b *Bindings) SlotFaults(slot uint32) (uint64, error) { b.mu.Lock() defer b.mu.Unlock() @@ -332,7 +424,7 @@ func (b *Bindings) SlotFaults(slot uint32) (uint64, error) { } // start writes and freezes the policy before any binding is written. -func (b *Bindings) start(policy, members, armed Map, faults FaultMap) error { +func (b *Bindings) start(policy, members, armed Map, faults, witness FaultMap) error { b.mu.Lock() defer b.mu.Unlock() if b.stopped { @@ -350,7 +442,7 @@ func (b *Bindings) start(policy, members, armed Map, faults FaultMap) error { if err := policy.Freeze(); err != nil { return fmt.Errorf("freezing policy: %w", err) } - b.members, b.armed, b.faults = members, armed, faults + b.members, b.armed, b.faults, b.witness = members, armed, faults, witness for _, c := range b.desired { if err := b.bind(c); err != nil && !errors.Is(err, ErrContainerGone) { return b.recordError(err) diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go index 8cf052426e..bdd06db910 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go @@ -75,7 +75,7 @@ func TestFreezeBeforeBindAndLifecycle(t *testing.T) { armed := &fakeMap{typ: ebpf.TaskStorage} var armedPaths []string b.armExisting = func(_ Map, path string) error { armedPaths = append(armedPaths, path); return nil } - if err := b.start(policy, members, armed, nil); err != nil { + if err := b.start(policy, members, armed, nil, nil); err != nil { t.Fatal(err) } if err := b.Ready(nil); err != nil { @@ -124,7 +124,7 @@ func TestFreezeBeforeBindAndLifecycle(t *testing.T) { func TestFailedArmingRemovesBinding(t *testing.T) { b, policy, members, _ := newFixture(t, func(Map) error { return nil }) b.armExisting = func(Map, string) error { return errors.New("no cgroup") } - if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}, nil); err == nil { + if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}, nil, nil); err == nil { t.Fatal("arming failure must fail startup") } if len(members.data) != 0 { @@ -139,7 +139,7 @@ func TestSameSlotRetryArmsAfterFailedArming(t *testing.T) { b.config.InitialBindings, b.desired = nil, map[execpolicy.Container]Binding{} arms := 0 b.armExisting = func(Map, string) error { arms++; return nil } - if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}, nil); err != nil { + if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}, nil, nil); err != nil { t.Fatal(err) } b.armExisting = func(Map, string) error { arms++; return errors.New("walk failed") } @@ -163,7 +163,7 @@ func TestSlotMoveRetryArmsAfterFailedRollback(t *testing.T) { b, policy, members, _ := newFixture(t, func(Map) error { return nil }) arms := 0 b.armExisting = func(Map, string) error { arms++; return nil } - if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}, nil); err != nil { + if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}, nil, nil); err != nil { t.Fatal(err) } b.armExisting = func(Map, string) error { @@ -187,11 +187,11 @@ func TestSlotMoveRetryArmsAfterFailedRollback(t *testing.T) { func TestRejectsWrongBindingLayoutAndPolicyErrors(t *testing.T) { b, policy, members, log := newFixture(t, func(Map) error { return nil }) members.max = 8 - if err := b.start(policy, members, nil, nil); err == nil || len(*log) != 0 { + if err := b.start(policy, members, nil, nil, nil); err == nil || len(*log) != 0 { t.Fatalf("wrong binding layout must fail before writes: %v %v", err, *log) } b, policy, members, log = newFixture(t, func(Map) error { return errors.New("bad layout") }) - if err := b.start(policy, members, nil, nil); err == nil || len(*log) != 0 { + if err := b.start(policy, members, nil, nil, nil); err == nil || len(*log) != 0 { t.Fatalf("policy writer error must stop startup: %v %v", err, *log) } } @@ -205,7 +205,7 @@ func TestNewValidates(t *testing.T) { func TestStartSkipsGoneContainer(t *testing.T) { b, policy, members, _ := newFixture(t, func(Map) error { return nil }) b.armExisting = func(Map, string) error { return ErrContainerGone } - if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}, nil); err != nil { + if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}, nil, nil); err != nil { t.Fatalf("a gone container must not fail startup: %v", err) } if err := b.Ready(nil); err != nil || len(members.data) != 0 { @@ -236,7 +236,7 @@ func TestSlotFaults(t *testing.T) { if _, err := b.SlotFaults(0); err == nil { t.Fatal("no fault map must be an error, not zero faults") } - if err := b.start(policy, members, nil, fakeFaults{1: 3}); err != nil { + if err := b.start(policy, members, nil, fakeFaults{1: 3}, nil); err != nil { t.Fatal(err) } if n, err := b.SlotFaults(0); err != nil || n != 0 { @@ -246,3 +246,77 @@ func TestSlotFaults(t *testing.T) { t.Fatalf("slot 1: %d %v", n, err) } } + +func TestBindingFlags(t *testing.T) { + log := &[]string{} + policy := &fakeMap{typ: ebpf.Hash, data: map[interface{}]interface{}{}, log: log, name: "policy"} + members := &fakeMap{typ: ebpf.Hash, key: 8, value: 8, max: MaxBindings, data: map[interface{}]interface{}{}, log: log, name: "bindings"} + b, err := New(Config{Name: "test", Generation: 5, Slots: 2, PolicyMap: "policy", WritePolicy: func(Map) error { return nil }, + StrictSlot: func(slot uint32) bool { return slot == 1 }}) + if err != nil { + t.Fatal(err) + } + if err := b.start(policy, members, nil, nil, nil); err != nil { + t.Fatal(err) + } + // Exact scope by default; the class follows the slot. + if err := b.Bind(Binding{Container: identity, Slot: 0}); err != nil || members.data[uint64(42)] != (bindingValue{Slot: 0}) { + t.Fatalf("audit exact binding: %v %v", err, members.data[uint64(42)]) + } + if err := b.Bind(Binding{Container: identity, Slot: 1}); err != nil || members.data[uint64(42)] != (bindingValue{Slot: 1, Flags: bindingStrict}) { + t.Fatalf("strict exact binding: %v %v", err, members.data[uint64(42)]) + } + // Same slot, new scope: the value is written again. + parent := ContainerIdentity{Runtime: "parent", ID: "p1", CgroupID: 7, CgroupPath: "/kubepods.slice/p1"} + if err := b.Bind(Binding{Container: parent, Slot: 1}); err != nil { + t.Fatal(err) + } + if err := b.Bind(Binding{Container: parent, Slot: 1, Subtree: true, RuntimeHelpers: true}); err != nil || + members.data[uint64(7)] != (bindingValue{Slot: 1, Flags: bindingStrict | bindingSubtree | bindingRuntimeHelpers}) { + t.Fatalf("strict subtree binding: %v %v", err, members.data[uint64(7)]) + } +} + +func TestBindingScopeValidation(t *testing.T) { + b, policy, members, _ := newFixture(t, func(Map) error { return nil }) + if err := b.start(policy, members, nil, nil, nil); err != nil { + t.Fatal(err) + } + if err := b.Bind(Binding{Container: identity, RuntimeHelpers: true}); err == nil { + t.Fatal("the runtime helper exemption on an exact binding must be refused") + } + deep := ContainerIdentity{Runtime: "parent", ID: "deep", CgroupID: 9, CgroupPath: "/" + strings.Repeat("d/", MaxSubtreeLevel) + "leaf"} + if err := b.Bind(Binding{Container: deep, Subtree: true}); err == nil { + t.Fatal("a subtree binding the program cannot see from its descendants must be refused") + } + if err := b.Bind(Binding{Container: deep}); err != nil { + t.Fatalf("an exact binding has no depth limit: %v", err) + } + if got := cgroupLevel("/a/b/"); got != 2 { + t.Fatalf("cgroupLevel: %d", got) + } +} + +type fakeWitness map[uint32]uint64 + +func (f fakeWitness) Lookup(k, v interface{}) error { + *v.(*uint64) = f[k.(uint32)] + return nil +} +func (fakeWitness) Type() ebpf.MapType { return ebpf.Array } +func (fakeWitness) KeySize() uint32 { return 4 } +func (fakeWitness) ValueSize() uint32 { return 8 } +func (fakeWitness) MaxEntries() uint32 { return armingWitnessCount } + +func TestArmingWitness(t *testing.T) { + b, policy, members, _ := newFixture(t, func(Map) error { return nil }) + if _, err := b.ArmingWitness(); err == nil { + t.Fatal("no witness map must be an error, not zero faults") + } + if err := b.start(policy, members, nil, nil, fakeWitness{0: 2, 1: 5}); err != nil { + t.Fatal(err) + } + if w, err := b.ArmingWitness(); err != nil || w != (ArmingWitness{NodeFaults: 2, ExemptExecs: 5}) { + t.Fatalf("witness: %+v %v", w, err) + } +} diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index e7b28e3bb4..0d04e97d64 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -86,34 +86,51 @@ policy: program outside the container runtime's mount namespace; children inherit the mark. So runc's own setup (namespaces, mounts, `pivot_root`, `setns` for `exec`) is never blocked, and `docker run` / `docker exec` keep working. - OCI `createContainer` and `startContainer` hooks are not armed either: a - task is not armed at exec when its parent is unarmed, outside the runtime - mount namespace, and in the same mount namespace and cgroup (runc init forks - the hooks inside the container). + OCI `createContainer` and `startContainer` hooks are not armed only under + an owner-admitted binding flag, `Binding.RuntimeHelpers` (needs `Subtree`): + the owner sets it on a parent cgroup bound before its containers start, + never on a container binding. Under that flag a task is not armed at exec + when its parent is unarmed, outside the runtime mount namespace, and in the + same mount namespace and cgroup (runc init forks the hooks inside the + container). Each exempted exec is counted (`Bindings.ArmingWitness`, + `ExemptExecs`). Without the flag there is no helper exemption, as in + enforce_exec's Strict profile. When a container is bound, the owner also arms the tasks that already run in it (pidfd writes to `armed_tasks`). The runtime mount namespace is the `host-mntns-id` gadget parameter; the owner sets it to the mount namespace of PID 1 as it sees it (in kind, the node container's). When it is 0, the program uses the global init task's. - **Lost arming.** The kernel can fail to allocate a task's mark (memory - pressure). When an armed task forks and the child's mark fails, an enforcing - slot (one with any enforce entry) refuses the fork (`-ENOMEM`): an unmarked - child would escape every rule. A clone into another cgroup - (`CLONE_INTO_CGROUP`) is refused the same way. An audit-only slot, or an - unbound cgroup, allows the fork. A new thread of an armed group leader is + pressure). When an armed task forks and the child's mark fails, a Strict + binding (its slot has any enforce entry) refuses the fork (`-ENOMEM`): an + unmarked child would escape every rule. Audit never blocks, even on a fault: + an Audit binding, or an unbound cgroup, allows the fork. When that fork is a + clone into another cgroup (`CLONE_INTO_CGROUP`), the child can start in a + Strict cgroup, so it counts a node fault (`Bindings.ArmingWitness`, + `NodeFaults`) that the owner reports against every Strict binding. A new thread of an armed group leader is covered through the leader, so it is never refused. An unarmed parent (for example a runtime helper) is out of scope, not a failure. Every failed mark that leaves a task unarmed is counted in `counters` slot 4. A failed mark of a new thread is not counted: the thread is covered through its leader, and it is marked at its own exec. Moving a container to another slot arms its running tasks again. -- **Tracking faults.** A failed mark in an enforcing slot (a refused fork, or a +- **Slot faults.** A failed mark in a Strict binding (a refused fork, or a lost mark at exec, which cannot be refused) is also counted per slot in `arming_faults`. The owner reads it with `Bindings.SlotFaults`: a nonzero count means the slot's protection is not confirmed. A new instance starts at zero and arms the running tasks again. Audit-only faults do not count there. -- **Scope.** A bound cgroup also covers up to 31 levels of child cgroups. This is - by design and differs from enforce_exec, which binds the exact cgroup only. +- **Missing slot policy.** A bound cgroup whose slot has no META entry is a + structural fault, not an unbound task. A Strict binding then denies every + judged check (`-EPERM`) and counts a slot fault in `arming_faults`; an + Audit binding allows. Both count in `counters` slot 4. The binding carries its class + (`BINDING_STRICT`), so the class is known without the policy. The owner sets + it from the slot's META mode. +- **Scope.** A binding covers its exact cgroup, as in enforce_exec. A binding + with `Subtree` (`BINDING_SUBTREE`) also covers its descendant cgroups at any + depth; the owner sets it only for a parent cgroup bound before its containers + start, and reports it. A subtree binding must be at cgroup level 31 or less + (the controller refuses a deeper one). When both apply, the task's own + binding wins, then the nearest subtree parent. - **Capabilities.** A `capable()` call is judged only when the task uses its own credentials (overlayfs copy-up uses the mounter's) and the call is not a no-audit probe. @@ -128,9 +145,9 @@ policy: ## Known limits -- **Runtime helpers are trusted.** A task forked by an unarmed task inside the - container (an OCI hook, or a process a hook leaves behind) is not judged, - and neither are its children. Only the runtime or a host root process can +- **Runtime helpers are trusted under `RuntimeHelpers`.** In a binding with the + flag, a task forked by an unarmed task inside the container (an OCI hook, or + a process a hook leaves behind) is not judged, and neither are its children. Only the runtime or a host root process can create such a task. This was verified with runc, which execs the workload and `exec` commands from runc init itself; a runtime that forks them from a helper inside the container would leave them unjudged. A binding (or @@ -143,14 +160,15 @@ policy: from the runtime's own setup, so the gadget does not retry it; a new instance arms it again. - **Fault injection.** The `arming-fail-fork-marks` parameter makes every child - mark fail at fork. It exists for tests (`runtime/runtime_kernel_test.go`); - owners do not set it. + mark fail at fork, and `policy-fail-headers` makes every slot miss its META + entry. They exist for tests (`runtime/runtime_kernel_test.go`); owners do + not set them. - **Kernels older than 6.1** are not supported yet. -- **Leaving the bound cgroup subtree.** A task that can write to a cgroup - outside its bound subtree (its `cgroup.procs`, or `clone3` with - `CLONE_INTO_CGROUP`) runs there without the policy. This needs write access - to the host cgroup tree, which a normal container does not have. A - `CLONE_INTO_CGROUP` clone is refused only when the child's mark fails. +- **Leaving the bound cgroup.** A task that can write to a cgroup outside its + binding's scope (its `cgroup.procs`, or `clone3` with `CLONE_INTO_CGROUP`) + runs there without the policy. This needs write access to the host cgroup + tree, which a normal container does not have. A `CLONE_INTO_CGROUP` clone is + refused only when the child's mark fails in a Strict binding. - **No-audit capability checks** (`CAP_OPT_NOAUDIT`) are excluded from capability enforcement. Most are kernel probes (for example the overcommit check), and judging them would deny normal work in every bound container. Some @@ -175,7 +193,7 @@ policy: | Map | Content | |---|---| | `policy` | frozen hash: `{slot, kind, id, qual, name}` → `{rank, rule_id, origin_id}`; each slot's META entry holds generation and mode (`privilege_policy.h`) | -| `bindings`, `armed_tasks`, `arming_faults`, `counters` | from the common library | +| `bindings`, `armed_tasks`, `arming_faults`, `arming_witness`, `counters` | from the common library; a `bindings` value is `{slot, flags}` (`BINDING_SUBTREE`, `BINDING_STRICT`, `BINDING_RUNTIME_HELPERS`) | The `verdict` datasource shares `generation`, `cgroup`, `deny`, `mode` and `host_pid` with enforce_exec and adds `rule_id`, `origin_id`, `is_default`, diff --git a/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml b/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml index 659bb5270b..58d3b5636d 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml +++ b/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml @@ -26,3 +26,7 @@ params: key: arming-fail-fork-marks defaultValue: "false" description: test only - fail every child arming mark at fork (fault injection) + policy_fail_headers: + key: policy-fail-headers + defaultValue: "false" + description: test only - every slot misses its policy header (fault injection) diff --git a/pkg/ebpf/gadgets/enforce_privilege/privilege_policy.h b/pkg/ebpf/gadgets/enforce_privilege/privilege_policy.h index ba0b8bf2d7..e84fa2e211 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/privilege_policy.h +++ b/pkg/ebpf/gadgets/enforce_privilege/privilege_policy.h @@ -10,12 +10,14 @@ #include #include -// A task that moves into a child cgroup of its container stays covered, up to -// this many levels below the bound cgroup. +// A subtree binding (a parent cgroup of containers) covers its descendant +// cgroups. Container bindings are exact. #define BINDINGS_CGROUP_DEPTH 32 #include "../common/bindings.h" #include "../common/policy.h" +#include "../common/verdict.h" +#include "../common/arming.h" #define PRIV_POLICY_MAX_ENTRIES 16384 #define PRIV_NAME_LEN 16 @@ -47,19 +49,39 @@ struct { __type(value, struct policy_value); } policy SEC(".maps"); +// Fault injection for tests only: every slot misses its META entry. Owners +// never set it in production. +const volatile bool policy_fail_headers = false; + // priv_header returns the META entry of the slot for a task in a bound cgroup, -// or NULL when nothing must be judged. +// or NULL when nothing must be judged. It also returns the binding flags. +// +// A bound cgroup whose slot has no META entry is a structural fault, not an +// unbound task: it counts as a section error. For a Strict binding it is also +// a slot fault, and *fault is true: the hook must deny. An Audit binding +// allows. static __always_inline struct policy_value *priv_header(__u64 *cgroup, - __u32 *slot) + __u32 *slot, + __u32 *flags, + bool *fault) { struct priv_key key = { .kind = PRIV_KIND_META }; struct policy_value *header; - if (!binding_lookup(cgroup, slot)) + *fault = false; + if (!binding_lookup(cgroup, slot, flags)) return NULL; key.slot = *slot; header = bpf_map_lookup_elem(&policy, &key); - if (!header || header->mode == POLICY_MODE_OFF) + if (!header || policy_fail_headers) { + verdict_count(VERDICT_COUNT_SECTION_ERROR); + if (*flags & BINDING_STRICT) { + arming_slot_fault(*slot); + *fault = true; + } + return NULL; + } + if (header->mode == POLICY_MODE_OFF) return NULL; return header; } diff --git a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c index 590b4fdd61..5ff3b84db4 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c +++ b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c @@ -29,14 +29,15 @@ #include +#include "privilege_policy.h" #include "../common/arming.h" #include "../common/verdict.h" -#include "privilege_policy.h" char LICENSE[] SEC("license") = "GPL"; GADGET_PARAM(host_mntns_id); GADGET_PARAM(arming_fail_fork_marks); +GADGET_PARAM(policy_fail_headers); #define EPERM 1 #define ENOMEM 12 @@ -395,14 +396,21 @@ static __always_inline void resolve_fstype(__u32 slot, const char *fstype, /* Scope and verdict */ // priv_scope returns the policy header for an armed task in a bound cgroup, -// or NULL. -static __always_inline struct policy_value *priv_scope(__u64 *cgid, __u32 *slot) +// or NULL. On NULL, return PRIV_NO_HEADER(fault): a Strict binding with no +// policy for its slot denies (see priv_header). +static __always_inline struct policy_value *priv_scope(__u64 *cgid, __u32 *slot, + bool *fault) { + __u32 flags = 0; + + *fault = false; if (!arming_task_armed(bpf_get_current_task_btf())) return NULL; - return priv_header(cgid, slot); + return priv_header(cgid, slot, &flags, fault); } +#define PRIV_NO_HEADER(fault) ((fault) ? -EPERM : 0) + static __always_inline int priv_verdict(void *ctx, enum priv_hook hook, const struct priv_target *t, const struct policy_value *header, @@ -456,61 +464,64 @@ static __always_inline int check_syscall(void *ctx, enum priv_hook hook, struct policy_match m = {}; __u64 cgid = 0; __u32 slot = 0; + bool fault; struct policy_value *header; if (sc == PRIV_SC_NONE) return 0; - header = priv_scope(&cgid, &slot); + header = priv_scope(&cgid, &slot, &fault); if (!header) - return 0; + return PRIV_NO_HEADER(fault); resolve_syscall(slot, sc, flags, &m); return priv_verdict(ctx, hook, &t, header, cgid, slot, &m); } /* Arming */ -// priv_enforcing_slot returns true, with the slot, when the current task is -// in a bound cgroup whose slot enforces. -static __always_inline bool priv_enforcing_slot(__u32 *slot) +// priv_strict_binding returns true, with the slot, when a Strict binding +// covers the current task. The class is in the binding, so it is known even +// when the slot has no policy. +static __always_inline bool priv_strict_binding(__u32 *slot, __u32 *flags) { - struct policy_value *header; __u64 cgid = 0; - header = priv_header(&cgid, slot); - return header && header->mode == POLICY_MODE_ENFORCE; + return binding_lookup(&cgid, slot, flags) && (*flags & BINDING_STRICT); } -// The exec hook cannot fail the exec. A lost mark in an enforcing slot is a +// The exec hook cannot fail the exec. A lost mark in a Strict binding is a // tracking fault: the owner reports the binding as not confirmed. SEC("lsm/bprm_committed_creds") int BPF_PROG(ig_priv_exec) { - __u32 slot = 0; + __u32 slot = 0, flags = 0; + bool strict = priv_strict_binding(&slot, &flags); - if (!arming_on_exec()) { + if (!arming_on_exec(flags)) { verdict_count(VERDICT_COUNT_SECTION_ERROR); - if (priv_enforcing_slot(&slot)) + if (strict) arming_slot_fault(slot); } return 0; } // priv_lost_child handles an armed parent whose child could not be marked. -// An unmarked child would escape every rule, so an enforcing slot refuses the -// fork and counts a tracking fault. An audit slot or an unbound cgroup allows -// it; the failure is in the section error counter only. A clone into another -// cgroup is refused too, because the child's binding is not known here. +// An unmarked child would escape every rule, so a Strict binding refuses the +// fork and counts a slot fault. Audit is nonblocking even on faults: an Audit +// binding or an unbound parent allows the fork. With CLONE_INTO_CGROUP the +// child can start in a Strict cgroup (its cgroup is not known here), so that +// case counts a node fault, which the owner reports against every Strict +// binding. static __always_inline int priv_lost_child(unsigned long clone_flags) { - __u32 slot = 0; + __u32 slot = 0, flags = 0; verdict_count(VERDICT_COUNT_SECTION_ERROR); - if (priv_enforcing_slot(&slot)) { + if (priv_strict_binding(&slot, &flags)) { arming_slot_fault(slot); return -ENOMEM; } if ((__u64)clone_flags & CLONE_INTO_CGROUP) - return -ENOMEM; + arming_witness_count(ARMING_WITNESS_NODE_FAULT); return 0; } @@ -523,11 +534,12 @@ static __always_inline int do_userns_create(void *ctx) struct policy_match m = {}; __u64 cgid = 0; __u32 slot = 0; + bool fault; struct policy_value *header; - header = priv_scope(&cgid, &slot); + header = priv_scope(&cgid, &slot, &fault); if (!header) - return 0; + return PRIV_NO_HEADER(fault); if (sc == PRIV_SC_UNSHARE || sc == PRIV_SC_CLONE || sc == PRIV_SC_CLONE3) { @@ -570,10 +582,11 @@ static __always_inline int do_task_alloc(void *ctx, struct task_struct *task, struct policy_value *header; __u64 cgid = 0; __u32 slot = 0; + bool fault; - header = priv_scope(&cgid, &slot); + header = priv_scope(&cgid, &slot, &fault); if (!header) - return 0; + return PRIV_NO_HEADER(fault); resolve_syscall(slot, PRIV_SC_CLONE, flags, &m); resolve_syscall(slot, PRIV_SC_CLONE3, flags, &m); return priv_verdict(ctx, PRIV_HOOK_TASK_ALLOC, &t, header, cgid, slot, &m); @@ -637,6 +650,7 @@ static __always_inline int do_capable(void *ctx, const struct cred *cred, struct policy_match m = {}; __u64 cgid = 0; __u32 slot = 0; + bool fault; struct policy_value *header; // Kernel probes that do not grant an action (for example the overcommit @@ -648,9 +662,9 @@ static __always_inline int do_capable(void *ctx, const struct cred *cred, // root filesystem. if ((void *)cred != (void *)task->real_cred) return 0; - header = priv_scope(&cgid, &slot); + header = priv_scope(&cgid, &slot, &fault); if (!header) - return 0; + return PRIV_NO_HEADER(fault); resolve_capability(slot, cap, &m); if (cap == CAP_SYS_ADMIN && current_is_compat()) { @@ -722,11 +736,12 @@ static __always_inline int do_sb_kern_mount(void *ctx, struct policy_match m = {}; __u64 cgid = 0; __u32 slot = 0; + bool fault; struct policy_value *header; - header = priv_scope(&cgid, &slot); + header = priv_scope(&cgid, &slot, &fault); if (!header) - return 0; + return PRIV_NO_HEADER(fault); bpf_probe_read_kernel_str(t.fstype, sizeof(t.fstype), BPF_CORE_READ(sb, s_type, name)); t.sc = priv_syscall_from_nr(current_syscall_nr()); @@ -813,15 +828,16 @@ static __always_inline int do_ptrace_access_check(void *ctx, unsigned int mode) struct policy_value *header; __u64 cgid = 0; __u32 slot = 0; + bool fault; if (current_is_compat()) { // Read-mode /proc access (ps, top) is never ptrace. if (!(mode & PTRACE_MODE_ATTACH)) return 0; // Unknown syscall: apply all three ptrace-family syscalls. - header = priv_scope(&cgid, &slot); + header = priv_scope(&cgid, &slot, &fault); if (!header) - return 0; + return PRIV_NO_HEADER(fault); resolve_syscall(slot, PRIV_SC_PTRACE, 0, &m); resolve_syscall(slot, PRIV_SC_PROCESS_VM_READV, 0, &m); resolve_syscall(slot, PRIV_SC_PROCESS_VM_WRITEV, 0, &m); diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go index 10d15d5fe2..ff144495ce 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go @@ -120,6 +120,18 @@ func writePolicy(entries []privpolicy.KernelEntry) func(enforcement.Map) error { } } +// strictSlots returns the class of each slot: a slot enforces when its META +// entry has the enforce mode (see ../privilege_policy.h). +func strictSlots(p privpolicy.Compiled) func(uint32) bool { + strict := map[uint32]bool{} + for _, e := range p.Entries { + if e.Key.Kind == privpolicy.KindMeta && e.Value.Mode == privpolicy.KernelModeEnforce { + strict[e.Key.Slot] = true + } + } + return func(slot uint32) bool { return strict[slot] } +} + type operator struct{} func New() operators.DataOperator { return &operator{} } @@ -158,6 +170,8 @@ func (*operator) InstantiateDataOperator(gc operators.GadgetContext, _ api.Param WritePolicy: writePolicy(config.Policy.Entries), ArmMap: "armed_tasks", FaultMap: "arming_faults", + WitnessMap: "arming_witness", + StrictSlot: strictSlots(config.Policy), }) if err != nil { return nil, err diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go index 5c5f4e839f..10def2907a 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go @@ -231,6 +231,60 @@ func TestForkMarkFaultInjection(t *testing.T) { if faults != 0 { t.Fatalf("audit faults must not block: %d", faults) } + // A plain fork keeps the child in the Audit cgroup: no + // node fault (only CLONE_INTO_CGROUP counts one). + if w, err := f.bindings.ArmingWitness(); err != nil || w.NodeFaults != 0 { + t.Fatalf("node faults: %+v %v", w, err) + } + } + }) + } +} + +// TestMissingSlotPolicyFailsClosed makes every slot miss its policy header. A +// bound cgroup with no policy is a structural fault: a Strict binding must deny +// every judged check and count a slot fault, and an Audit binding must allow +// and count no slot fault. +func TestMissingSlotPolicyFailsClosed(t *testing.T) { + for _, mode := range []string{"enforce", "audit"} { + t.Run(mode, func(t *testing.T) { + f := newKernelFixture(t) + // The policy allows unshare; only the missing header can deny it. + rule := privpolicy.Rule{Name: "no-modules", Action: "deny", Target: privpolicy.Target{Syscall: []string{"init_module"}}} + doc := privpolicy.Document{Version: 1, Name: "headers-" + mode, Policy: privpolicy.Policy{Mode: mode, Privilege: privpolicy.Section{Default: "allow", Rules: []privpolicy.Rule{rule}}}} + compiled, err := privpolicy.CompileKernel(doc, 1) + if err != nil { + t.Fatal(err) + } + f.start(t, "privilege-headers-"+mode, compiled, map[string]string{"policy-fail-headers": "true"}) + unshare := func() error { return f.exec("unshare", "-m", "true") } + if err := unshare(); err != nil { + t.Fatalf("an unbound container must not be judged: %v", err) + } + if err := f.bindings.Bind(enforcement.Binding{Container: f.container}); err != nil { + t.Fatal(err) + } + defer f.bindings.Unbind(f.container) + err = unshare() + faults, ferr := f.bindings.SlotFaults(0) + if ferr != nil { + t.Fatal(ferr) + } + switch mode { + case "enforce": + if err == nil { + t.Fatal("a Strict binding with no slot policy must deny") + } + if faults == 0 { + t.Fatal("the structural fault must count a slot fault") + } + case "audit": + if err != nil { + t.Fatalf("an Audit binding must allow even on a fault: %v", err) + } + if faults != 0 { + t.Fatalf("audit faults must not count as slot faults: %d", faults) + } } }) } @@ -309,6 +363,123 @@ func TestEarlierBPFDenyHolds(t *testing.T) { } } +// standInTaskAlloc loads a BPF LSM task_alloc program that stands in for +// another enforcement gadget on the same hook (enforce_exec owns task_alloc in +// its shared collection). It returns errno (0 or a negative errno) for forks +// from the cgroup cgid and 0 elsewhere, and ignores the previous verdict. +func standInTaskAlloc(t *testing.T, cgid uint64, errno int32) *ebpf.Program { + t.Helper() + prog, err := ebpf.NewProgram(&ebpf.ProgramSpec{ + Type: ebpf.LSM, AttachType: ebpf.AttachLSMMac, AttachTo: "task_alloc", License: "GPL", + Instructions: asm.Instructions{ + asm.FnGetCurrentCgroupId.Call(), + asm.LoadImm(asm.R1, int64(cgid), asm.DWord), + asm.JNE.Reg(asm.R0, asm.R1, "other"), + asm.Mov.Imm(asm.R0, errno), + asm.Return(), + asm.Mov.Imm(asm.R0, 0).WithSymbol("other"), + asm.Return(), + }, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { prog.Close() }) + return prog +} + +// TestTaskAllocCoexistence attaches a second BPF LSM program on task_alloc, in +// both attach orders, next to the gadget with every child mark failing: +// - an Audit binding never denies a fork, even on that tracking fault; +// - the other program's fork deny is never overridden: in Audit the fork +// fails with its errno, in Strict it fails (with the errno of whichever +// program runs first; the kernel runs the newest link first and stops at +// the first nonzero return). +// +// The stand-in has no task state. The state check that Matthias's review asks +// for (a privilege deny before and after enforce_exec allocates its lineage +// state, with reclaim in task_free) needs enforce_exec itself on the same node. +func TestTaskAllocCoexistence(t *testing.T) { + for _, order := range []string{"other-older", "other-newer"} { + for _, mode := range []string{"enforce", "audit"} { + t.Run(order+"/"+mode, func(t *testing.T) { + f := newKernelFixture(t) + fork := func() string { + out, err := exec.CommandContext(f.ctx, "docker", "exec", f.id, "sh", "-c", "true & wait").CombinedOutput() + switch { + case err == nil: + return "ok" + case strings.Contains(string(out), "Permission denied"): + return "EACCES" + } + return "failed: " + strings.TrimSpace(string(out)) + } + attach := func(prog *ebpf.Program) link.Link { + l, err := link.AttachLSM(link.LSMOptions{Program: prog}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { l.Close() }) + return l + } + allow, deny := standInTaskAlloc(t, f.container.CgroupID, 0), standInTaskAlloc(t, f.container.CgroupID, -13) + var other link.Link + if order == "other-older" { + other = attach(allow) + } + rule := privpolicy.Rule{Name: "no-modules", Action: "deny", Target: privpolicy.Target{Syscall: []string{"init_module"}}} + doc := privpolicy.Document{Version: 1, Name: "coexist-" + mode, Policy: privpolicy.Policy{Mode: mode, Privilege: privpolicy.Section{Default: "allow", Rules: []privpolicy.Rule{rule}}}} + compiled, err := privpolicy.CompileKernel(doc, 1) + if err != nil { + t.Fatal(err) + } + f.start(t, "privilege-coexist", compiled, map[string]string{"arming-fail-fork-marks": "true"}) + if order == "other-newer" { + other = attach(allow) + } + if err := f.bindings.Bind(enforcement.Binding{Container: f.container}); err != nil { + t.Fatal(err) + } + defer f.bindings.Unbind(f.container) + + got := fork() + if mode == "audit" && got != "ok" { + t.Fatalf("an Audit binding must never deny a fork: %s", got) + } + if mode == "enforce" && got == "ok" { + t.Fatal("a Strict binding must refuse a fork whose child cannot be armed") + } + + // Replace the allowing program with a denying one, in the + // same attach order. + other.Close() + if order == "other-older" { + // Older than the gadget: attach a new gadget after it. + f.bindings.Unbind(f.container) + attach(deny) + f.start(t, "privilege-coexist-2", compiled, map[string]string{"arming-fail-fork-marks": "true"}) + if err := f.bindings.Bind(enforcement.Binding{Container: f.container}); err != nil { + t.Fatal(err) + } + } else { + attach(deny) + } + got = fork() + switch mode { + case "audit": + if got != "EACCES" { + t.Fatalf("the other program's deny must hold next to an Audit binding: %s", got) + } + case "enforce": + if got == "ok" { + t.Fatal("a deny must never become an allow") + } + } + }) + } + } +} + // hookFixture is a runc bundle whose OCI hooks run chroot, and a parent cgroup // (the cgroup of a group of containers) that is bound before the container // starts, with a policy that denies chroot. @@ -320,8 +491,9 @@ type hookFixture struct { // newHookFixture needs runc, PRIVILEGE_ROOTFS (a rootfs folder with busybox sh // and chroot; it is copied) and util-linux nsenter on the host. extra are ebpf -// gadget params. -func newHookFixture(t *testing.T, extra map[string]string) *hookFixture { +// gadget params. scope sets the Subtree and RuntimeHelpers flags of the parent +// cgroup's binding. +func newHookFixture(t *testing.T, extra map[string]string, scope enforcement.Binding) *hookFixture { t.Helper() if os.Getenv("PRIVILEGE_KERNEL_TEST") != "1" || os.Getenv("PRIVILEGE_ROOTFS") == "" { t.Skip("explicit disposable kernel fixture opt-in and PRIVILEGE_ROOTFS required") @@ -392,7 +564,8 @@ func newHookFixture(t *testing.T, extra map[string]string) *hookFixture { t.Fatal(err) } h.start(t, "privilege-hooks", compiled, extra) - if err := h.bindings.Bind(enforcement.Binding{Container: parentID}); err != nil { + scope.Container = parentID + if err := h.bindings.Bind(scope); err != nil { t.Fatal(err) } t.Cleanup(func() { h.bindings.Unbind(parentID) }) @@ -435,18 +608,76 @@ func (h *hookFixture) workloadDenied(t *testing.T) { } } +// parentScope is the binding of a parent cgroup bound before its containers +// start: it covers them, and it admits the runtime helper exemption. +var parentScope = enforcement.Binding{Subtree: true, RuntimeHelpers: true} + +// chrootVerdicts counts the chroot verdict events so far. +func (f *kernelFixture) chrootVerdicts() int { + f.mu.Lock() + defer f.mu.Unlock() + n := 0 + for _, e := range f.events { + if strings.Contains(e, "chroot") { + n++ + } + } + return n +} + +// TestRuntimeHooksJudgedWithoutExemption: a subtree binding without +// RuntimeHelpers has no helper exemption, so both OCI hooks are judged. Their +// chroot is denied (the hooks ignore the result, so the container starts) and +// no exec is exempted. +func TestRuntimeHooksJudgedWithoutExemption(t *testing.T) { + h := newHookFixture(t, nil, enforcement.Binding{Subtree: true}) + if out, err := h.startContainer(t); err != nil { + t.Fatalf("container start: %v %s", err, out) + } + h.workloadDenied(t) + time.Sleep(500 * time.Millisecond) + // Two hooks and the workload. + if n := h.chrootVerdicts(); n < 3 { + t.Fatalf("both hooks must be judged with no exemption: %d chroot verdicts, %v", n, h.events) + } + if w, err := h.bindings.ArmingWitness(); err != nil || w.ExemptExecs != 0 { + t.Fatalf("no exec may be exempted: %+v %v", w, err) + } +} + +// TestExactScopeSkipsDescendants: a binding without Subtree covers its own +// cgroup only, so the container in a child cgroup is not judged. +func TestExactScopeSkipsDescendants(t *testing.T) { + h := newHookFixture(t, nil, enforcement.Binding{}) + if out, err := h.startContainer(t); err != nil { + t.Fatalf("container start: %v %s", err, out) + } + deadline := time.Now().Add(10 * time.Second) + var rc []byte + for time.Now().Before(deadline) && len(rc) == 0 { + rc, _ = os.ReadFile(filepath.Join(h.bundle, "rootfs", "tmp", "workload-rc")) + time.Sleep(100 * time.Millisecond) + } + if strings.TrimSpace(string(rc)) != "0" { + t.Fatalf("an exact parent binding must not judge a child cgroup, rc=%q", rc) + } +} + // TestRuntimeHooksAreNotArmed: the OCI createContainer and startContainer // hooks (and their children) are runtime setup, so they must not be judged // and the container must start. The workload, a "runc exec" command and every // task they start must be judged. func TestRuntimeHooksAreNotArmed(t *testing.T) { - h := newHookFixture(t, nil) + h := newHookFixture(t, nil, parentScope) f, parent, id, run := h.kernelFixture, h.parent, h.id, h.run if out, err := h.startContainer(t); err != nil { t.Fatalf("the hooks must not be judged, so the container must start: %v %s", err, out) } h.workloadDenied(t) + if w, err := h.bindings.ArmingWitness(); err != nil || w.ExemptExecs == 0 { + t.Errorf("the exempted hook execs must be counted: %+v %v", w, err) + } if out, err := run("runc", "exec", id, "sh", "-c", "unshare -m true && true"); err != nil { t.Fatalf("an allowed exec must work: %v %s", err, out) } @@ -507,7 +738,7 @@ func TestRuntimeHooksAreNotArmed(t *testing.T) { // namespace and cgroup. The workload is in the container mount namespace and // its parent (runc) is not, so it must still be armed and judged. func TestWrongRuntimeNamespaceStillJudgesWorkload(t *testing.T) { - h := newHookFixture(t, map[string]string{"host-mntns-id": "1"}) + h := newHookFixture(t, map[string]string{"host-mntns-id": "1"}, parentScope) if out, err := h.startContainer(t); err != nil { t.Logf("container start with a wrong runtime namespace: %v %s", err, out) return // judging the runtime itself fails closed diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go index bd923b502f..e55000fadf 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go @@ -104,3 +104,18 @@ func TestHooksMatchProgram(t *testing.T) { t.Errorf("program has %d int hooks, Hooks has %d", len(seen), len(Hooks)) } } + +func TestStrictSlots(t *testing.T) { + enforce := privpolicy.Document{Version: 1, Policy: privpolicy.Policy{Mode: "enforce", Privilege: privpolicy.Section{Default: "allow", + Rules: []privpolicy.Rule{{Name: "r", Action: "deny", Target: privpolicy.Target{Syscall: []string{"mount"}}}}}}} + audit := enforce + audit.Policy.Mode = "audit" + c, err := privpolicy.CompileSlots([][]privpolicy.Document{{audit}, {enforce}}, 5) + if err != nil { + t.Fatal(err) + } + strict := strictSlots(c) + if strict(0) || !strict(1) || strict(2) { + t.Fatalf("strict slots: %v %v %v", strict(0), strict(1), strict(2)) + } +} From 8af40f72ba7b40b3ddf8b39c5c40c51f17c4b8cb Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:45:01 +0200 Subject: [PATCH 29/34] refactor(enforce_privilege): export StrictSlots for the owner's health check Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go | 6 +++--- pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go index ff144495ce..301219fde3 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go @@ -120,9 +120,9 @@ func writePolicy(entries []privpolicy.KernelEntry) func(enforcement.Map) error { } } -// strictSlots returns the class of each slot: a slot enforces when its META +// StrictSlots returns the class of each slot: a slot enforces when its META // entry has the enforce mode (see ../privilege_policy.h). -func strictSlots(p privpolicy.Compiled) func(uint32) bool { +func StrictSlots(p privpolicy.Compiled) func(uint32) bool { strict := map[uint32]bool{} for _, e := range p.Entries { if e.Key.Kind == privpolicy.KindMeta && e.Value.Mode == privpolicy.KernelModeEnforce { @@ -171,7 +171,7 @@ func (*operator) InstantiateDataOperator(gc operators.GadgetContext, _ api.Param ArmMap: "armed_tasks", FaultMap: "arming_faults", WitnessMap: "arming_witness", - StrictSlot: strictSlots(config.Policy), + StrictSlot: StrictSlots(config.Policy), }) if err != nil { return nil, err diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go index e55000fadf..21ce7bfaf2 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go @@ -114,7 +114,7 @@ func TestStrictSlots(t *testing.T) { if err != nil { t.Fatal(err) } - strict := strictSlots(c) + strict := StrictSlots(c) if strict(0) || !strict(1) || strict(2) { t.Fatalf("strict slots: %v %v %v", strict(0), strict(1), strict(2)) } From 9388578be65dc26eea8432ddaef08bc401337c1d Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:49:48 +0200 Subject: [PATCH 30/34] fix(enforcement): deny a task outside a Strict exact binding's scope Review follow-ups: - A task that moves into a child cgroup of a Strict exact binding is no longer unjudged. binding_lookup returns the nearer Strict exact parent with BINDING_OUTSIDE_SCOPE, and the privilege header treats it as a structural fault (deny, slot fault). - A lost child in a slot whose policy enforces counts as Strict even when the owner set no class, and the controller requires the slot class when it has a fault map. - The deepest subtree level is a per-gadget Config field (0 refuses Subtree bindings). - The task_alloc coexistence test judges only the shell's forks and checks the exact errno of the program that decides. A new kernel test moves a task into a child cgroup of a Strict binding. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/common/bindings.h | 14 ++- .../gadgets/common/enforcement/bindings.go | 24 +++-- .../common/enforcement/bindings_test.go | 33 ++++++- pkg/ebpf/gadgets/enforce_privilege/README.md | 23 +++-- .../enforce_privilege/privilege_policy.h | 5 +- .../gadgets/enforce_privilege/program.bpf.c | 13 ++- .../enforce_privilege/runtime/runtime.go | 1 + .../runtime/runtime_kernel_test.go | 88 +++++++++++++++++-- 8 files changed, 172 insertions(+), 29 deletions(-) diff --git a/pkg/ebpf/gadgets/common/bindings.h b/pkg/ebpf/gadgets/common/bindings.h index a85b55bbee..4767080165 100644 --- a/pkg/ebpf/gadgets/common/bindings.h +++ b/pkg/ebpf/gadgets/common/bindings.h @@ -36,6 +36,11 @@ #define BINDING_STRICT (1U << 1) // the slot enforces: a fault denies // Owner-admitted runtime helper exemption (see arming_on_exec in arming.h). #define BINDING_RUNTIME_HELPERS (1U << 2) +// Set by binding_lookup only, never stored: the task is in a descendant cgroup +// of a Strict exact binding. The binding does not cover it, but a Strict +// workload that moved itself into a child cgroup must not run unjudged, so the +// caller treats it as a structural fault (deny). +#define BINDING_OUTSIDE_SCOPE (1U << 31) struct binding_value { __u32 slot; @@ -51,7 +56,9 @@ struct { // binding_lookup returns true when a binding covers the current task: the // binding of the task's own cgroup, or else the nearest parent binding with -// BINDING_SUBTREE. It returns the bound cgroup id, the slot and the flags. +// BINDING_SUBTREE. It returns the bound cgroup id, the slot and the flags. A +// Strict exact parent that is nearer than any subtree parent is returned with +// BINDING_OUTSIDE_SCOPE added. static __always_inline bool binding_lookup(__u64 *cgroup, __u32 *slot, __u32 *flags) { @@ -72,7 +79,8 @@ static __always_inline bool binding_lookup(__u64 *cgroup, __u32 *slot, break; parent_id = bpf_get_current_ancestor_cgroup_id(i); parent = bpf_map_lookup_elem(&bindings, &parent_id); - if (parent && (parent->flags & BINDING_SUBTREE)) { + if (parent && (parent->flags & + (BINDING_SUBTREE | BINDING_STRICT))) { value = parent; id = parent_id; } @@ -84,6 +92,8 @@ static __always_inline bool binding_lookup(__u64 *cgroup, __u32 *slot, *cgroup = id; *slot = value->slot; *flags = value->flags; + if (id != bpf_get_current_cgroup_id() && !(*flags & BINDING_SUBTREE)) + *flags |= BINDING_OUTSIDE_SCOPE; return true; } diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings.go b/pkg/ebpf/gadgets/common/enforcement/bindings.go index 4052dd2682..41b88d4fb0 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings.go @@ -50,9 +50,9 @@ const ( bindingRuntimeHelpers uint32 = 1 << 2 ) -// MaxSubtreeLevel is the deepest cgroup level of a Subtree binding -// (BINDINGS_CGROUP_DEPTH - 1 in the gadgets that cover subtrees). -const MaxSubtreeLevel = 31 +// SubtreeLevels is Config.SubtreeLevels for a gadget built with +// BINDINGS_CGROUP_DEPTH 32: a Subtree binding up to cgroup level 31. +const SubtreeLevels = 31 // bindingValue matches struct binding_value in ../bindings.h. type bindingValue struct { @@ -97,8 +97,16 @@ type Config struct { WitnessMap string // StrictSlot reports whether a slot enforces. The class goes into each // binding (BINDING_STRICT), so the program fails closed for a Strict - // binding even when the slot's policy is missing. Nil: no slot enforces. + // binding even when the slot's policy is missing. Required with FaultMap. StrictSlot func(slot uint32) bool + // SubtreeLevels is the deepest cgroup level of a Subtree binding that the + // program sees from its descendants: BINDINGS_CGROUP_DEPTH - 1 of the + // gadget. 0 refuses Subtree bindings. + // + // Levels are counted in the owner's cgroup namespace. The kernel counts + // from the host root, so an owner in a nested cgroup namespace must not + // bind a subtree near this limit. + SubtreeLevels int } // FaultMap is the part of *ebpf.Map that SlotFaults reads. @@ -147,6 +155,10 @@ func New(config Config) (*Bindings, error) { if len(config.InitialBindings) > MaxBindings { return nil, fmt.Errorf("too many initial bindings") } + if config.FaultMap != "" && config.StrictSlot == nil { + // Without the class a Strict binding would allow on a missing slot. + return nil, fmt.Errorf("enforcement controller with a fault map requires the slot class") + } b := &Bindings{config: config, desired: map[execpolicy.Container]Binding{}, active: map[uint64]Binding{}, unarmed: map[uint64]bool{}, armExisting: ArmExisting} for _, c := range config.InitialBindings { if err := b.validate(c); err != nil { @@ -223,9 +235,9 @@ func (b *Bindings) validate(c Binding) error { if c.RuntimeHelpers && !c.Subtree { return fmt.Errorf("the runtime helper exemption needs a subtree binding") } - if c.Subtree && cgroupLevel(c.Container.CgroupPath) > MaxSubtreeLevel { + if c.Subtree && cgroupLevel(c.Container.CgroupPath) > b.config.SubtreeLevels { // The program would not see the binding from its descendants. - return fmt.Errorf("subtree binding deeper than cgroup level %d", MaxSubtreeLevel) + return fmt.Errorf("subtree binding deeper than cgroup level %d", b.config.SubtreeLevels) } return validate(c.Container) } diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go index bdd06db910..3e147fec17 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go @@ -63,7 +63,7 @@ func newFixture(t *testing.T, write func(Map) error) (*Bindings, *fakeMap, *fake log := &[]string{} policy := &fakeMap{typ: ebpf.Hash, data: map[interface{}]interface{}{}, log: log, name: "policy"} members := &fakeMap{typ: ebpf.Hash, key: 8, value: 8, max: MaxBindings, data: map[interface{}]interface{}{}, log: log, name: "bindings"} - b, err := New(Config{Name: "test", Generation: 5, Slots: 2, PolicyMap: "policy", WritePolicy: write, InitialBindings: []Binding{{Container: identity}}}) + b, err := New(Config{Name: "test", Generation: 5, Slots: 2, PolicyMap: "policy", WritePolicy: write, InitialBindings: []Binding{{Container: identity}}, SubtreeLevels: SubtreeLevels}) if err != nil { t.Fatal(err) } @@ -252,7 +252,7 @@ func TestBindingFlags(t *testing.T) { policy := &fakeMap{typ: ebpf.Hash, data: map[interface{}]interface{}{}, log: log, name: "policy"} members := &fakeMap{typ: ebpf.Hash, key: 8, value: 8, max: MaxBindings, data: map[interface{}]interface{}{}, log: log, name: "bindings"} b, err := New(Config{Name: "test", Generation: 5, Slots: 2, PolicyMap: "policy", WritePolicy: func(Map) error { return nil }, - StrictSlot: func(slot uint32) bool { return slot == 1 }}) + StrictSlot: func(slot uint32) bool { return slot == 1 }, SubtreeLevels: SubtreeLevels}) if err != nil { t.Fatal(err) } @@ -285,10 +285,14 @@ func TestBindingScopeValidation(t *testing.T) { if err := b.Bind(Binding{Container: identity, RuntimeHelpers: true}); err == nil { t.Fatal("the runtime helper exemption on an exact binding must be refused") } - deep := ContainerIdentity{Runtime: "parent", ID: "deep", CgroupID: 9, CgroupPath: "/" + strings.Repeat("d/", MaxSubtreeLevel) + "leaf"} + deep := ContainerIdentity{Runtime: "parent", ID: "deep", CgroupID: 9, CgroupPath: "/" + strings.Repeat("d/", SubtreeLevels) + "leaf"} if err := b.Bind(Binding{Container: deep, Subtree: true}); err == nil { t.Fatal("a subtree binding the program cannot see from its descendants must be refused") } + limit := ContainerIdentity{Runtime: "parent", ID: "limit", CgroupID: 10, CgroupPath: "/" + strings.Repeat("d/", SubtreeLevels-1) + "leaf"} + if err := b.Bind(Binding{Container: limit, Subtree: true}); err != nil { + t.Fatalf("a subtree binding at level %d must be accepted: %v", SubtreeLevels, err) + } if err := b.Bind(Binding{Container: deep}); err != nil { t.Fatalf("an exact binding has no depth limit: %v", err) } @@ -320,3 +324,26 @@ func TestArmingWitness(t *testing.T) { t.Fatalf("witness: %+v %v", w, err) } } + +func TestSubtreeNeedsLevels(t *testing.T) { + log := &[]string{} + policy := &fakeMap{typ: ebpf.Hash, data: map[interface{}]interface{}{}, log: log, name: "policy"} + members := &fakeMap{typ: ebpf.Hash, key: 8, value: 8, max: MaxBindings, data: map[interface{}]interface{}{}, log: log, name: "bindings"} + b, err := New(Config{Name: "exact-only", Generation: 5, Slots: 1, PolicyMap: "policy", WritePolicy: func(Map) error { return nil }}) + if err != nil { + t.Fatal(err) + } + if err := b.start(policy, members, nil, nil, nil); err != nil { + t.Fatal(err) + } + parent := ContainerIdentity{Runtime: "parent", ID: "p1", CgroupID: 7, CgroupPath: "/kubepods.slice/p1"} + if err := b.Bind(Binding{Container: parent, Subtree: true}); err == nil { + t.Fatal("a gadget that checks no parent cgroup must refuse a subtree binding") + } +} + +func TestFaultMapNeedsSlotClass(t *testing.T) { + if _, err := New(Config{Name: "test", Generation: 5, Slots: 1, PolicyMap: "policy", WritePolicy: func(Map) error { return nil }, FaultMap: "arming_faults"}); err == nil { + t.Fatal("a fault map without the slot class must be refused") + } +} diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index 0d04e97d64..398d602492 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -129,8 +129,13 @@ policy: with `Subtree` (`BINDING_SUBTREE`) also covers its descendant cgroups at any depth; the owner sets it only for a parent cgroup bound before its containers start, and reports it. A subtree binding must be at cgroup level 31 or less - (the controller refuses a deeper one). When both apply, the task's own - binding wins, then the nearest subtree parent. + (the controller refuses a deeper one; levels are counted in the owner's + cgroup namespace, the kernel counts from the host root). The task's own + binding wins, then the nearest parent that is a subtree binding or a Strict + binding. A task in a child cgroup of a Strict exact binding (a workload that + moved itself there) is outside the binding's scope but is not left + unjudged: every judged check denies, as a structural fault with a slot + fault. - **Capabilities.** A `capable()` call is judged only when the task uses its own credentials (overlayfs copy-up uses the mounter's) and the call is not a no-audit probe. @@ -147,7 +152,10 @@ policy: - **Runtime helpers are trusted under `RuntimeHelpers`.** In a binding with the flag, a task forked by an unarmed task inside the container (an OCI hook, or - a process a hook leaves behind) is not judged, and neither are its children. Only the runtime or a host root process can + a process a hook leaves behind) is not judged, and neither are its children. + A workload task whose exec mark failed (memory pressure) is also an unarmed + parent; in an Audit binding that loss counts no slot fault, and its exec'd + children are then exempt too. `ExemptExecs` makes every exemption visible. Only the runtime or a host root process can create such a task. This was verified with runc, which execs the workload and `exec` commands from runc init itself; a runtime that forks them from a helper inside the container would leave them unjudged. A binding (or @@ -165,10 +173,15 @@ policy: not set them. - **Kernels older than 6.1** are not supported yet. - **Leaving the bound cgroup.** A task that can write to a cgroup outside its - binding's scope (its `cgroup.procs`, or `clone3` with `CLONE_INTO_CGROUP`) - runs there without the policy. This needs write access to the host cgroup + binding's subtree (its `cgroup.procs`, or `clone3` with `CLONE_INTO_CGROUP`) + runs there without this binding's policy. A child cgroup of a Strict binding + denies (see Scope); a cgroup elsewhere needs write access to the host cgroup tree, which a normal container does not have. A `CLONE_INTO_CGROUP` clone is refused only when the child's mark fails in a Strict binding. +- **Node faults stay counted.** `NodeFaults` only grows in one instance, so + one lost `CLONE_INTO_CGROUP` mark marks every Strict binding unconfirmed. + The owner recovers with a new instance, which starts at zero and arms the + running tasks again. - **No-audit capability checks** (`CAP_OPT_NOAUDIT`) are excluded from capability enforcement. Most are kernel probes (for example the overcommit check), and judging them would deny normal work in every bound container. Some diff --git a/pkg/ebpf/gadgets/enforce_privilege/privilege_policy.h b/pkg/ebpf/gadgets/enforce_privilege/privilege_policy.h index e84fa2e211..330ae814a6 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/privilege_policy.h +++ b/pkg/ebpf/gadgets/enforce_privilege/privilege_policy.h @@ -59,7 +59,8 @@ const volatile bool policy_fail_headers = false; // A bound cgroup whose slot has no META entry is a structural fault, not an // unbound task: it counts as a section error. For a Strict binding it is also // a slot fault, and *fault is true: the hook must deny. An Audit binding -// allows. +// allows. A task outside a Strict binding's exact scope (BINDING_OUTSIDE_SCOPE) +// is the same fault. static __always_inline struct policy_value *priv_header(__u64 *cgroup, __u32 *slot, __u32 *flags, @@ -73,7 +74,7 @@ static __always_inline struct policy_value *priv_header(__u64 *cgroup, return NULL; key.slot = *slot; header = bpf_map_lookup_elem(&policy, &key); - if (!header || policy_fail_headers) { + if (!header || policy_fail_headers || (*flags & BINDING_OUTSIDE_SCOPE)) { verdict_count(VERDICT_COUNT_SECTION_ERROR); if (*flags & BINDING_STRICT) { arming_slot_fault(*slot); diff --git a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c index 5ff3b84db4..63f6736f63 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c +++ b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c @@ -480,12 +480,21 @@ static __always_inline int check_syscall(void *ctx, enum priv_hook hook, // priv_strict_binding returns true, with the slot, when a Strict binding // covers the current task. The class is in the binding, so it is known even -// when the slot has no policy. +// when the slot has no policy. A slot whose policy enforces is Strict too, so +// an owner that sets no class never makes a fault path allow. static __always_inline bool priv_strict_binding(__u32 *slot, __u32 *flags) { + struct priv_key key = { .kind = PRIV_KIND_META }; + struct policy_value *header; __u64 cgid = 0; - return binding_lookup(&cgid, slot, flags) && (*flags & BINDING_STRICT); + if (!binding_lookup(&cgid, slot, flags)) + return false; + if (*flags & BINDING_STRICT) + return true; + key.slot = *slot; + header = bpf_map_lookup_elem(&policy, &key); + return header && header->mode == POLICY_MODE_ENFORCE; } // The exec hook cannot fail the exec. A lost mark in a Strict binding is a diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go index 301219fde3..a3dd4ce163 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go @@ -172,6 +172,7 @@ func (*operator) InstantiateDataOperator(gc operators.GadgetContext, _ api.Param FaultMap: "arming_faults", WitnessMap: "arming_witness", StrictSlot: StrictSlots(config.Policy), + SubtreeLevels: enforcement.SubtreeLevels, }) if err != nil { return nil, err diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go index 10def2907a..139e8dcbf4 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go @@ -290,6 +290,61 @@ func TestMissingSlotPolicyFailsClosed(t *testing.T) { } } +// TestChildCgroupOfStrictBindingFailsClosed moves a task of a Strict exact +// binding into a child cgroup of the container. The binding does not cover +// the child cgroup (exact scope), but the task must not run unjudged: every +// judged check is denied as a structural fault. +func TestChildCgroupOfStrictBindingFailsClosed(t *testing.T) { + f := newKernelFixture(t) + rule := privpolicy.Rule{Name: "no-modules", Action: "deny", Target: privpolicy.Target{Syscall: []string{"init_module"}}} + doc := privpolicy.Document{Version: 1, Name: "moved", Policy: privpolicy.Policy{Mode: "enforce", Privilege: privpolicy.Section{Default: "allow", Rules: []privpolicy.Rule{rule}}}} + compiled, err := privpolicy.CompileKernel(doc, 1) + if err != nil { + t.Fatal(err) + } + f.start(t, "privilege-moved", compiled, nil) + if err := f.bindings.Bind(enforcement.Binding{Container: f.container}); err != nil { + t.Fatal(err) + } + defer f.bindings.Unbind(f.container) + if err := f.exec("unshare", "-m", "true"); err != nil { + t.Fatalf("the policy allows unshare in the bound cgroup: %v", err) + } + parent := filepath.Join("/sys/fs/cgroup", f.container.CgroupPath) + child := filepath.Join(parent, "moved") + if err := os.Mkdir(child, 0o755); err != nil { + t.Fatal(err) + } + defer os.Remove(child) + cmd := exec.CommandContext(f.ctx, "docker", "exec", f.id, "sh", "-c", "sleep 2; unshare -m true; echo rc=$?") + var out strings.Builder + cmd.Stdout, cmd.Stderr = &out, &out + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + // Move the shell into the child cgroup while it sleeps. + moved := false + for deadline := time.Now().Add(1500 * time.Millisecond); !moved && time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) { + procs, _ := os.ReadFile(filepath.Join(parent, "cgroup.procs")) + for _, pid := range strings.Fields(string(procs)) { + cmdline, _ := os.ReadFile(filepath.Join("/proc", pid, "cmdline")) + if strings.Contains(string(cmdline), "unshare -m true; echo") { + moved = os.WriteFile(filepath.Join(child, "cgroup.procs"), []byte(pid), 0o644) == nil + } + } + } + if !moved { + t.Fatal("could not move the shell into the child cgroup") + } + cmd.Wait() + if !strings.Contains(out.String(), "rc=1") { + t.Fatalf("unshare in a child cgroup of a Strict exact binding must be denied: %q", out.String()) + } + if faults, err := f.bindings.SlotFaults(0); err != nil || faults == 0 { + t.Fatalf("the task outside the exact scope must count a slot fault: %d %v", faults, err) + } +} + // TestEarlierBPFDenyHolds checks that the gadget never turns another BPF LSM // program's deny into an allow, in both attach orders. The kernel runs the // programs of a hook newest first and stops at the first nonzero return. The @@ -366,7 +421,8 @@ func TestEarlierBPFDenyHolds(t *testing.T) { // standInTaskAlloc loads a BPF LSM task_alloc program that stands in for // another enforcement gadget on the same hook (enforce_exec owns task_alloc in // its shared collection). It returns errno (0 or a negative errno) for forks -// from the cgroup cgid and 0 elsewhere, and ignores the previous verdict. +// by a task named "sh" in the cgroup cgid, and 0 elsewhere (so runc init's own +// clones pass), and ignores the previous verdict. func standInTaskAlloc(t *testing.T, cgid uint64, errno int32) *ebpf.Program { t.Helper() prog, err := ebpf.NewProgram(&ebpf.ProgramSpec{ @@ -375,6 +431,13 @@ func standInTaskAlloc(t *testing.T, cgid uint64, errno int32) *ebpf.Program { asm.FnGetCurrentCgroupId.Call(), asm.LoadImm(asm.R1, int64(cgid), asm.DWord), asm.JNE.Reg(asm.R0, asm.R1, "other"), + // comm into the stack; "sh\0\0" is 0x6873 as a little-endian word. + asm.Mov.Reg(asm.R1, asm.R10), + asm.Add.Imm(asm.R1, -16), + asm.Mov.Imm(asm.R2, 16), + asm.FnGetCurrentComm.Call(), + asm.LoadMem(asm.R1, asm.R10, -16, asm.Word), + asm.JNE.Imm(asm.R1, 0x6873, "other"), asm.Mov.Imm(asm.R0, errno), asm.Return(), asm.Mov.Imm(asm.R0, 0).WithSymbol("other"), @@ -392,9 +455,10 @@ func standInTaskAlloc(t *testing.T, cgid uint64, errno int32) *ebpf.Program { // both attach orders, next to the gadget with every child mark failing: // - an Audit binding never denies a fork, even on that tracking fault; // - the other program's fork deny is never overridden: in Audit the fork -// fails with its errno, in Strict it fails (with the errno of whichever -// program runs first; the kernel runs the newest link first and stops at -// the first nonzero return). +// fails with its errno; in Strict it fails with the errno of the program +// that runs first (the kernel runs the newest link first and stops at the +// first nonzero return): EACCES when the other program is newer, ENOMEM +// (the gadget's lost child) when it is older. // // The stand-in has no task state. The state check that Matthias's review asks // for (a privilege deny before and after enforce_exec allocates its lineage @@ -411,6 +475,8 @@ func TestTaskAllocCoexistence(t *testing.T) { return "ok" case strings.Contains(string(out), "Permission denied"): return "EACCES" + case strings.Contains(string(out), "Out of memory"), strings.Contains(string(out), "Cannot allocate memory"): + return "ENOMEM" } return "failed: " + strings.TrimSpace(string(out)) } @@ -440,14 +506,13 @@ func TestTaskAllocCoexistence(t *testing.T) { if err := f.bindings.Bind(enforcement.Binding{Container: f.container}); err != nil { t.Fatal(err) } - defer f.bindings.Unbind(f.container) got := fork() if mode == "audit" && got != "ok" { t.Fatalf("an Audit binding must never deny a fork: %s", got) } - if mode == "enforce" && got == "ok" { - t.Fatal("a Strict binding must refuse a fork whose child cannot be armed") + if mode == "enforce" && got != "ENOMEM" { + t.Fatalf("a Strict binding must refuse a fork whose child cannot be armed: %s", got) } // Replace the allowing program with a denying one, in the @@ -471,10 +536,15 @@ func TestTaskAllocCoexistence(t *testing.T) { t.Fatalf("the other program's deny must hold next to an Audit binding: %s", got) } case "enforce": - if got == "ok" { - t.Fatal("a deny must never become an allow") + want := "EACCES" + if order == "other-older" { + want = "ENOMEM" + } + if got != want { + t.Fatalf("the newest program decides and no deny becomes an allow: got %s, want %s", got, want) } } + f.bindings.Unbind(f.container) }) } } From 9f9dfde62038678de1e4470b16794f660767e349 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:52:07 +0200 Subject: [PATCH 31/34] test(enforce_privilege): a Strict exact parent denies its descendants; README known limit Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/common/bindings.h | 4 ++- pkg/ebpf/gadgets/enforce_privilege/README.md | 5 ++++ .../runtime/runtime_kernel_test.go | 27 +++++++++---------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/pkg/ebpf/gadgets/common/bindings.h b/pkg/ebpf/gadgets/common/bindings.h index 4767080165..c499231c98 100644 --- a/pkg/ebpf/gadgets/common/bindings.h +++ b/pkg/ebpf/gadgets/common/bindings.h @@ -73,7 +73,9 @@ static __always_inline bool binding_lookup(__u64 *cgroup, __u32 *slot, __u64 parent_id; // From the top down, so a subtree binding covers a task at any - // depth below it. The last match is the nearest parent. + // depth below it. The last match is the nearest parent. A Strict + // exact binding above level BINDINGS_CGROUP_DEPTH - 1 is not seen + // from its descendants. for (int i = 1; i < BINDINGS_CGROUP_DEPTH; i++) { if (i >= level) break; diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index 398d602492..7e6958d990 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -178,6 +178,11 @@ policy: denies (see Scope); a cgroup elsewhere needs write access to the host cgroup tree, which a normal container does not have. A `CLONE_INTO_CGROUP` clone is refused only when the child's mark fails in a Strict binding. +- **Containers that move into their own child cgroup.** docker:dind moves its + processes into a child cgroup at start, and systemd images move PID 1 into + `init.scope`. Under a Strict exact binding every judged check of those tasks + is denied (see Scope), so such a container does not run. It needs an + explicit `Subtree` binding. - **Node faults stay counted.** `NodeFaults` only grows in one instance, so one lost `CLONE_INTO_CGROUP` mark marks every Strict binding unconfirmed. The owner recovers with a new instance, which starts at zero and arms the diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go index 139e8dcbf4..2bbc86a058 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go @@ -328,8 +328,9 @@ func TestChildCgroupOfStrictBindingFailsClosed(t *testing.T) { procs, _ := os.ReadFile(filepath.Join(parent, "cgroup.procs")) for _, pid := range strings.Fields(string(procs)) { cmdline, _ := os.ReadFile(filepath.Join("/proc", pid, "cmdline")) - if strings.Contains(string(cmdline), "unshare -m true; echo") { - moved = os.WriteFile(filepath.Join(child, "cgroup.procs"), []byte(pid), 0o644) == nil + if strings.Contains(string(cmdline), "unshare -m true; echo") && os.WriteFile(filepath.Join(child, "cgroup.procs"), []byte(pid), 0o644) == nil { + moved = true + break } } } @@ -451,7 +452,8 @@ func standInTaskAlloc(t *testing.T, cgid uint64, errno int32) *ebpf.Program { return prog } -// TestTaskAllocCoexistence attaches a second BPF LSM program on task_alloc, in +// TestTaskAllocCoexistence needs a busybox (or alpine) sh in the fixture +// container: its fork errors name the errno. It attaches a second BPF LSM program on task_alloc, in // both attach orders, next to the gadget with every child mark failing: // - an Audit binding never denies a fork, even on that tracking fault; // - the other program's fork deny is never overridden: in Audit the fork @@ -715,21 +717,18 @@ func TestRuntimeHooksJudgedWithoutExemption(t *testing.T) { } } -// TestExactScopeSkipsDescendants: a binding without Subtree covers its own -// cgroup only, so the container in a child cgroup is not judged. -func TestExactScopeSkipsDescendants(t *testing.T) { +// TestExactStrictParentDeniesDescendants: a Strict binding without Subtree +// covers its own cgroup only. The container in a child cgroup is outside its +// scope, so its judged checks are denied as a structural fault (with a slot +// fault), never allowed. +func TestExactStrictParentDeniesDescendants(t *testing.T) { h := newHookFixture(t, nil, enforcement.Binding{}) if out, err := h.startContainer(t); err != nil { t.Fatalf("container start: %v %s", err, out) } - deadline := time.Now().Add(10 * time.Second) - var rc []byte - for time.Now().Before(deadline) && len(rc) == 0 { - rc, _ = os.ReadFile(filepath.Join(h.bundle, "rootfs", "tmp", "workload-rc")) - time.Sleep(100 * time.Millisecond) - } - if strings.TrimSpace(string(rc)) != "0" { - t.Fatalf("an exact parent binding must not judge a child cgroup, rc=%q", rc) + h.workloadDenied(t) + if faults, err := h.bindings.SlotFaults(0); err != nil || faults == 0 { + t.Fatalf("a task outside the exact scope must count a slot fault: %d %v", faults, err) } } From 16ecd638dab7b7fc3705b326e0694ef9d7eb0c9a Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:29:29 +0200 Subject: [PATCH 32/34] fix(enforcement): the nearest bound parent decides; refuse a Strict binding too deep to see A task in a child cgroup of an Audit exact binding is no longer judged by a Strict binding further up: the nearest bound parent decides, and an Audit exact parent covers nothing. The controller refuses a Strict binding deeper than the walk, whose moved tasks the program could not see. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/common/bindings.h | 27 ++++++++++++------- .../gadgets/common/enforcement/bindings.go | 6 +++++ .../common/enforcement/bindings_test.go | 21 +++++++++++++++ 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/pkg/ebpf/gadgets/common/bindings.h b/pkg/ebpf/gadgets/common/bindings.h index c499231c98..5c3c689295 100644 --- a/pkg/ebpf/gadgets/common/bindings.h +++ b/pkg/ebpf/gadgets/common/bindings.h @@ -55,10 +55,10 @@ struct { } bindings SEC(".maps"); // binding_lookup returns true when a binding covers the current task: the -// binding of the task's own cgroup, or else the nearest parent binding with -// BINDING_SUBTREE. It returns the bound cgroup id, the slot and the flags. A -// Strict exact parent that is nearer than any subtree parent is returned with -// BINDING_OUTSIDE_SCOPE added. +// binding of the task's own cgroup, or else the nearest bound parent when it +// has BINDING_SUBTREE. When the nearest bound parent is a Strict exact +// binding, it is returned with BINDING_OUTSIDE_SCOPE added; an Audit exact +// one covers nothing (Audit never blocks). static __always_inline bool binding_lookup(__u64 *cgroup, __u32 *slot, __u32 *flags) { @@ -73,16 +73,15 @@ static __always_inline bool binding_lookup(__u64 *cgroup, __u32 *slot, __u64 parent_id; // From the top down, so a subtree binding covers a task at any - // depth below it. The last match is the nearest parent. A Strict - // exact binding above level BINDINGS_CGROUP_DEPTH - 1 is not seen + // depth below it. The last match is the nearest bound parent. A + // binding deeper than level BINDINGS_CGROUP_DEPTH - 1 is not seen // from its descendants. for (int i = 1; i < BINDINGS_CGROUP_DEPTH; i++) { if (i >= level) break; parent_id = bpf_get_current_ancestor_cgroup_id(i); parent = bpf_map_lookup_elem(&bindings, &parent_id); - if (parent && (parent->flags & - (BINDING_SUBTREE | BINDING_STRICT))) { + if (parent) { value = parent; id = parent_id; } @@ -91,11 +90,19 @@ static __always_inline bool binding_lookup(__u64 *cgroup, __u32 *slot, #endif if (!value) return false; + if (id != bpf_get_current_cgroup_id() && !(value->flags & BINDING_SUBTREE)) { + // The nearest bound parent is exact: an Audit one does not cover + // the task; a Strict one marks it outside its scope. + if (!(value->flags & BINDING_STRICT)) + return false; + *cgroup = id; + *slot = value->slot; + *flags = value->flags | BINDING_OUTSIDE_SCOPE; + return true; + } *cgroup = id; *slot = value->slot; *flags = value->flags; - if (id != bpf_get_current_cgroup_id() && !(*flags & BINDING_SUBTREE)) - *flags |= BINDING_OUTSIDE_SCOPE; return true; } diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings.go b/pkg/ebpf/gadgets/common/enforcement/bindings.go index 41b88d4fb0..6a6a414489 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings.go @@ -239,6 +239,12 @@ func (b *Bindings) validate(c Binding) error { // The program would not see the binding from its descendants. return fmt.Errorf("subtree binding deeper than cgroup level %d", b.config.SubtreeLevels) } + if b.config.SubtreeLevels > 0 && b.config.StrictSlot != nil && b.config.StrictSlot(c.Slot) && + cgroupLevel(c.Container.CgroupPath) > b.config.SubtreeLevels { + // A task that moves into a child cgroup would not be seen as + // outside the Strict binding's scope, so it would not be judged. + return fmt.Errorf("Strict binding deeper than cgroup level %d", b.config.SubtreeLevels) + } return validate(c.Container) } diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go index 3e147fec17..c30e9ec90a 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go @@ -347,3 +347,24 @@ func TestFaultMapNeedsSlotClass(t *testing.T) { t.Fatal("a fault map without the slot class must be refused") } } + +func TestStrictBindingDepth(t *testing.T) { + log := &[]string{} + policy := &fakeMap{typ: ebpf.Hash, data: map[interface{}]interface{}{}, log: log, name: "policy"} + members := &fakeMap{typ: ebpf.Hash, key: 8, value: 8, max: MaxBindings, data: map[interface{}]interface{}{}, log: log, name: "bindings"} + b, err := New(Config{Name: "test", Generation: 5, Slots: 2, PolicyMap: "policy", WritePolicy: func(Map) error { return nil }, + StrictSlot: func(slot uint32) bool { return slot == 1 }, SubtreeLevels: SubtreeLevels}) + if err != nil { + t.Fatal(err) + } + if err := b.start(policy, members, nil, nil, nil); err != nil { + t.Fatal(err) + } + deep := ContainerIdentity{Runtime: "docker", ID: "deep", CgroupID: 9, CgroupPath: "/" + strings.Repeat("d/", SubtreeLevels) + "leaf"} + if err := b.Bind(Binding{Container: deep, Slot: 1}); err == nil { + t.Fatal("a Strict binding whose descendants the program cannot see must be refused") + } + if err := b.Bind(Binding{Container: deep, Slot: 0}); err != nil { + t.Fatalf("an Audit exact binding has no depth limit: %v", err) + } +} From 12b3bf4133e7ccb9124240fef8cc26f7f9ba0286 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:04:53 +0200 Subject: [PATCH 33/34] feat(enforce_privilege): export operational faults as privilege verdicts A structural fault of an armed task (a bound slot with no policy, a task outside a Strict exact scope) and a lost tracking mark in a Strict binding send a privilege verdict with hook fault and the fault kind, the slot and the slot's generation, like the file section. Strict faults that deny are enforce verdicts; an Audit structural fault and a lost mark at exec are audit verdicts. The kernel tests check both modes. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/enforce_privilege/README.md | 7 ++ .../gadgets/enforce_privilege/gadget.yaml | 3 + .../enforce_privilege/privilege_policy.h | 32 +++++++-- .../gadgets/enforce_privilege/program.bpf.c | 71 +++++++++++++++++-- .../runtime/runtime_kernel_test.go | 31 ++++++++ 5 files changed, 132 insertions(+), 12 deletions(-) diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index 7e6958d990..2402ff9c8d 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -125,6 +125,13 @@ policy: Audit binding allows. Both count in `counters` slot 4. The binding carries its class (`BINDING_STRICT`), so the class is known without the policy. The owner sets it from the slot's META mode. +- **Fault verdicts.** A structural fault of an armed task, and a lost mark in + a Strict binding, also send a verdict with hook `fault` (`fault_raw`: + `structural` or `tracking`), the slot and the slot's generation (0 when the + slot has no policy). It is an enforce verdict when the hook denies and an + audit verdict when it allows (an Audit binding, or an exec, which cannot be + refused). A lost mark in an Audit binding sends none. The controller exports + it with reason `operational_error`, as the file section does. - **Scope.** A binding covers its exact cgroup, as in enforce_exec. A binding with `Subtree` (`BINDING_SUBTREE`) also covers its descendant cgroups at any depth; the owner sets it only for a parent cgroup bound before its containers diff --git a/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml b/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml index 58d3b5636d..2daabc1443 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml +++ b/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml @@ -13,6 +13,9 @@ datasources: syscall_raw: annotations: columns.hidden: true + fault_raw: + annotations: + columns.hidden: true containers: annotations: cli.default-output-mode: none diff --git a/pkg/ebpf/gadgets/enforce_privilege/privilege_policy.h b/pkg/ebpf/gadgets/enforce_privilege/privilege_policy.h index 330ae814a6..f3ad0b437d 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/privilege_policy.h +++ b/pkg/ebpf/gadgets/enforce_privilege/privilege_policy.h @@ -32,6 +32,13 @@ enum priv_kind : __u32 { PRIV_KIND_MOUNT_FSTYPE = 4, // name: filesystem type }; +enum priv_fault_kind : __u32 { + // A bound slot with no policy, or a task outside a Strict exact scope. + PRIV_FAULT_STRUCTURAL = 0, + // A lost arming mark in a Strict binding. + PRIV_FAULT_TRACKING = 1, +}; + struct priv_key { __u32 slot; __u32 kind; @@ -57,25 +64,31 @@ const volatile bool policy_fail_headers = false; // or NULL when nothing must be judged. It also returns the binding flags. // // A bound cgroup whose slot has no META entry is a structural fault, not an -// unbound task: it counts as a section error. For a Strict binding it is also -// a slot fault, and *fault is true: the hook must deny. An Audit binding -// allows. A task outside a Strict binding's exact scope (BINDING_OUTSIDE_SCOPE) -// is the same fault. +// unbound task: it counts as a section error, and *faulted is true with the +// slot's generation (0 when it has no policy), for the caller to report. For +// a Strict binding it is also a slot fault, and *fault is true: the hook must +// deny. An Audit binding allows. A task outside a Strict binding's exact +// scope (BINDING_OUTSIDE_SCOPE) is the same fault. static __always_inline struct policy_value *priv_header(__u64 *cgroup, __u32 *slot, __u32 *flags, - bool *fault) + bool *fault, + bool *faulted, + __u64 *generation) { struct priv_key key = { .kind = PRIV_KIND_META }; struct policy_value *header; *fault = false; + *faulted = false; if (!binding_lookup(cgroup, slot, flags)) return NULL; key.slot = *slot; header = bpf_map_lookup_elem(&policy, &key); if (!header || policy_fail_headers || (*flags & BINDING_OUTSIDE_SCOPE)) { verdict_count(VERDICT_COUNT_SECTION_ERROR); + *faulted = true; + *generation = header ? header->generation : 0; if (*flags & BINDING_STRICT) { arming_slot_fault(*slot); *fault = true; @@ -87,6 +100,15 @@ static __always_inline struct policy_value *priv_header(__u64 *cgroup, return header; } +// priv_slot_generation is the slot's generation, or 0 when it has no policy. +static __always_inline __u64 priv_slot_generation(__u32 slot) +{ + struct priv_key key = { .slot = slot, .kind = PRIV_KIND_META }; + struct policy_value *header = bpf_map_lookup_elem(&policy, &key); + + return header ? header->generation : 0; +} + static __always_inline void priv_consider(const struct priv_key *key, struct policy_match *m) { diff --git a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c index 63f6736f63..88934000e7 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c +++ b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c @@ -116,6 +116,8 @@ enum priv_hook : __u32 { PRIV_HOOK_TASK_FIX_SETUID, PRIV_HOOK_TASK_FIX_SETGID, PRIV_HOOK_CAPSET, + // An operational fault (fault_raw holds the kind). + PRIV_HOOK_FAULT, }; struct event { @@ -136,6 +138,8 @@ struct event { char fstype[PRIV_NAME_LEN]; // Policy slot (combination of policies) of the bound container. __u32 slot; + // Kind of an operational fault (hook fault only). + enum priv_fault_kind fault_raw; }; struct { @@ -144,6 +148,43 @@ struct { } events SEC(".maps"); GADGET_TRACER(verdict, events, event); +// priv_fault_report exports an operational fault as a privilege verdict (hook +// fault, the kind in fault_raw): enforce when the hook denies, audit when it +// allows (an Audit binding, or an exec that cannot be refused). generation is +// the slot's, so the verdict is attributed to the slot's policies (0 when the +// slot has no policy). +static __always_inline void priv_fault_report(__u64 cgid, __u32 slot, + __u64 generation, bool denied, + enum priv_fault_kind kind) +{ + struct event *ev; + + if (denied) + verdict_count(VERDICT_COUNT_DENIED); + if (!verdict_report_allowed()) { + verdict_count(VERDICT_COUNT_RATE_LIMITED); + return; + } + ev = bpf_ringbuf_reserve(&events, sizeof(*ev), 0); + if (!ev) { + verdict_count(VERDICT_COUNT_DROPPED); + return; + } + __builtin_memset(ev, 0, sizeof(*ev)); + ev->generation = generation; + ev->cgroup = cgid; + ev->deny = 1; + ev->mode = denied ? POLICY_MODE_ENFORCE : POLICY_MODE_AUDIT; + ev->host_pid = bpf_get_current_pid_tgid() >> 32; + ev->slot = slot; + ev->capability = -1; + bpf_get_current_comm(ev->comm, sizeof(ev->comm)); + ev->hook_raw = PRIV_HOOK_FAULT; + ev->fault_raw = kind; + bpf_ringbuf_submit(ev, 0); + verdict_count(VERDICT_COUNT_SUBMITTED); +} + // What a hook tried to do. Used for the event only. struct priv_target { enum priv_syscall sc; @@ -403,10 +444,18 @@ static __always_inline struct policy_value *priv_scope(__u64 *cgid, __u32 *slot, { __u32 flags = 0; + struct policy_value *header; + __u64 generation = 0; + bool faulted = false; + *fault = false; if (!arming_task_armed(bpf_get_current_task_btf())) return NULL; - return priv_header(cgid, slot, &flags, fault); + header = priv_header(cgid, slot, &flags, fault, &faulted, &generation); + if (faulted) + priv_fault_report(*cgid, *slot, generation, *fault, + PRIV_FAULT_STRUCTURAL); + return header; } #define PRIV_NO_HEADER(fault) ((fault) ? -EPERM : 0) @@ -482,13 +531,13 @@ static __always_inline int check_syscall(void *ctx, enum priv_hook hook, // covers the current task. The class is in the binding, so it is known even // when the slot has no policy. A slot whose policy enforces is Strict too, so // an owner that sets no class never makes a fault path allow. -static __always_inline bool priv_strict_binding(__u32 *slot, __u32 *flags) +static __always_inline bool priv_strict_binding(__u64 *cgid, __u32 *slot, + __u32 *flags) { struct priv_key key = { .kind = PRIV_KIND_META }; struct policy_value *header; - __u64 cgid = 0; - if (!binding_lookup(&cgid, slot, flags)) + if (!binding_lookup(cgid, slot, flags)) return false; if (*flags & BINDING_STRICT) return true; @@ -503,12 +552,17 @@ SEC("lsm/bprm_committed_creds") int BPF_PROG(ig_priv_exec) { __u32 slot = 0, flags = 0; - bool strict = priv_strict_binding(&slot, &flags); + __u64 cgid = 0; + bool strict = priv_strict_binding(&cgid, &slot, &flags); if (!arming_on_exec(flags)) { verdict_count(VERDICT_COUNT_SECTION_ERROR); - if (strict) + if (strict) { + // Counted and reported; the exec itself cannot fail. arming_slot_fault(slot); + priv_fault_report(cgid, slot, priv_slot_generation(slot), + false, PRIV_FAULT_TRACKING); + } } return 0; } @@ -523,10 +577,13 @@ int BPF_PROG(ig_priv_exec) static __always_inline int priv_lost_child(unsigned long clone_flags) { __u32 slot = 0, flags = 0; + __u64 cgid = 0; verdict_count(VERDICT_COUNT_SECTION_ERROR); - if (priv_strict_binding(&slot, &flags)) { + if (priv_strict_binding(&cgid, &slot, &flags)) { arming_slot_fault(slot); + priv_fault_report(cgid, slot, priv_slot_generation(slot), true, + PRIV_FAULT_TRACKING); return -ENOMEM; } if ((__u64)clone_flags & CLONE_INTO_CGROUP) diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go index 2bbc86a058..6b4bed611e 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go @@ -224,6 +224,10 @@ func TestForkMarkFaultInjection(t *testing.T) { if faults == 0 { t.Fatal("the refused fork must count a tracking fault") } + time.Sleep(500 * time.Millisecond) + if f.faultVerdicts(2) == 0 { + t.Fatalf("the refused fork must send an enforce fault verdict: %v", f.events) + } case "audit": if err != nil { t.Fatalf("audit mode must allow the fork: %v", err) @@ -231,6 +235,10 @@ func TestForkMarkFaultInjection(t *testing.T) { if faults != 0 { t.Fatalf("audit faults must not block: %d", faults) } + time.Sleep(500 * time.Millisecond) + if n := f.faultVerdicts(1) + f.faultVerdicts(2); n != 0 { + t.Fatalf("a lost Audit mark sends no fault verdict: %v", f.events) + } // A plain fork keeps the child in the Audit cgroup: no // node fault (only CLONE_INTO_CGROUP counts one). if w, err := f.bindings.ArmingWitness(); err != nil || w.NodeFaults != 0 { @@ -278,6 +286,10 @@ func TestMissingSlotPolicyFailsClosed(t *testing.T) { if faults == 0 { t.Fatal("the structural fault must count a slot fault") } + time.Sleep(500 * time.Millisecond) + if f.faultVerdicts(2) == 0 { + t.Fatalf("the denied fault must send an enforce fault verdict: %v", f.events) + } case "audit": if err != nil { t.Fatalf("an Audit binding must allow even on a fault: %v", err) @@ -285,6 +297,10 @@ func TestMissingSlotPolicyFailsClosed(t *testing.T) { if faults != 0 { t.Fatalf("audit faults must not count as slot faults: %d", faults) } + time.Sleep(500 * time.Millisecond) + if f.faultVerdicts(1) == 0 { + t.Fatalf("the allowed fault must send an audit fault verdict: %v", f.events) + } } }) } @@ -684,6 +700,21 @@ func (h *hookFixture) workloadDenied(t *testing.T) { // start: it covers them, and it admits the runtime helper exemption. var parentScope = enforcement.Binding{Subtree: true, RuntimeHelpers: true} +// faultVerdicts counts the operational fault verdicts (hook fault) with this +// mode so far. +func (f *kernelFixture) faultVerdicts(mode uint32) int { + f.mu.Lock() + defer f.mu.Unlock() + n := 0 + for _, e := range f.events { + hook := strings.ToLower(strings.Fields(e)[0]) + if strings.HasSuffix(hook, "fault") && strings.Contains(e, fmt.Sprintf(" mode=%d ", mode)) { + n++ + } + } + return n +} + // chrootVerdicts counts the chroot verdict events so far. func (f *kernelFixture) chrootVerdicts() int { f.mu.Lock() From 52fcdda4645008f92b52910c2594e2b7b6a55ef7 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:13:08 +0200 Subject: [PATCH 34/34] fix(enforce_privilege): clear every verdict record; wait for fault verdicts A rule verdict now clears its ring buffer record, so the new fault_raw field never carries bytes of an older record. The kernel tests poll for a fault verdict with a deadline instead of one fixed sleep. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/common/verdict.h | 2 +- .../gadgets/enforce_privilege/program.bpf.c | 1 + .../runtime/runtime_kernel_test.go | 19 +++++++++++++------ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/pkg/ebpf/gadgets/common/verdict.h b/pkg/ebpf/gadgets/common/verdict.h index 39d267af0b..9df84bb572 100644 --- a/pkg/ebpf/gadgets/common/verdict.h +++ b/pkg/ebpf/gadgets/common/verdict.h @@ -38,7 +38,7 @@ enum verdict_counter { VERDICT_COUNT_SUBMITTED = 1, VERDICT_COUNT_DROPPED = 2, VERDICT_COUNT_ALLOWED = 3, - VERDICT_COUNT_SECTION_ERROR = 4, // privilege: failed arming marks + VERDICT_COUNT_SECTION_ERROR = 4, // failed arming marks, structural faults VERDICT_COUNT_RATE_LIMITED = 5, VERDICT_COUNT_MAX = 6, }; diff --git a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c index 88934000e7..95f3a11b69 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c +++ b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c @@ -479,6 +479,7 @@ static __always_inline int priv_verdict(void *ctx, enum priv_hook hook, reporting = verdict_report_allowed(); e = reporting ? bpf_ringbuf_reserve(&events, sizeof(*e), 0) : NULL; if (e) { + __builtin_memset(e, 0, sizeof(*e)); VERDICT_FILL_BASE(e, m, header->generation, cgid); e->host_pid = bpf_get_current_pid_tgid() >> 32; e->rule_id = m->rule_id; diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go index 6b4bed611e..ac1204538c 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go @@ -224,8 +224,7 @@ func TestForkMarkFaultInjection(t *testing.T) { if faults == 0 { t.Fatal("the refused fork must count a tracking fault") } - time.Sleep(500 * time.Millisecond) - if f.faultVerdicts(2) == 0 { + if !f.waitFaultVerdict(2) { t.Fatalf("the refused fork must send an enforce fault verdict: %v", f.events) } case "audit": @@ -286,8 +285,7 @@ func TestMissingSlotPolicyFailsClosed(t *testing.T) { if faults == 0 { t.Fatal("the structural fault must count a slot fault") } - time.Sleep(500 * time.Millisecond) - if f.faultVerdicts(2) == 0 { + if !f.waitFaultVerdict(2) { t.Fatalf("the denied fault must send an enforce fault verdict: %v", f.events) } case "audit": @@ -297,8 +295,7 @@ func TestMissingSlotPolicyFailsClosed(t *testing.T) { if faults != 0 { t.Fatalf("audit faults must not count as slot faults: %d", faults) } - time.Sleep(500 * time.Millisecond) - if f.faultVerdicts(1) == 0 { + if !f.waitFaultVerdict(1) { t.Fatalf("the allowed fault must send an audit fault verdict: %v", f.events) } } @@ -715,6 +712,16 @@ func (f *kernelFixture) faultVerdicts(mode uint32) int { return n } +// waitFaultVerdict waits up to 5 s for a fault verdict with this mode. +func (f *kernelFixture) waitFaultVerdict(mode uint32) bool { + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) { + if f.faultVerdicts(mode) > 0 { + return true + } + } + return false +} + // chrootVerdicts counts the chroot verdict events so far. func (f *kernelFixture) chrootVerdicts() int { f.mu.Lock()