From c49672396d1cc71a213dfd7df6cce26a62f9cc46 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Mon, 2 Nov 2026 18:16:31 +0200 Subject: [PATCH 01/54] feat(sandboxpolicy): drive the privilege gadget from the sandbox policy controller - CompiledPolicy carries the privilege section; the compiler accepts it - SectionBackend splits each plan per section; plans without a privilege section go to the exec backend unchanged - PrivilegeBackend owns one enforce_privilege instance per node: live bind/unbind; a new policy combination replaces the instance with overlap - slots are created when a matching pod is scheduled to the node - sandboxPolicyPrivilegeImage enables the section; empty rejects it Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- cmd/main.go | 3 +- pkg/config/config.go | 2 + pkg/sandboxpolicy/compiler.go | 17 +- pkg/sandboxpolicy/controller.go | 50 +++ pkg/sandboxpolicy/igbackend/privilege.go | 360 ++++++++++++++++++ .../igbackend/privilege_kernel_test.go | 134 +++++++ .../igbackend/privilege_native.go | 140 +++++++ pkg/sandboxpolicy/igbackend/privilege_test.go | 216 +++++++++++ pkg/sandboxpolicy/integration/bindings.go | 40 ++ pkg/sandboxpolicy/integration/service.go | 56 ++- pkg/sandboxpolicy/sections.go | 255 +++++++++++++ pkg/sandboxpolicy/sections_test.go | 248 ++++++++++++ pkg/sandboxpolicy/types.go | 9 + 13 files changed, 1521 insertions(+), 9 deletions(-) create mode 100644 pkg/sandboxpolicy/igbackend/privilege.go create mode 100644 pkg/sandboxpolicy/igbackend/privilege_kernel_test.go create mode 100644 pkg/sandboxpolicy/igbackend/privilege_native.go create mode 100644 pkg/sandboxpolicy/igbackend/privilege_test.go create mode 100644 pkg/sandboxpolicy/sections.go create mode 100644 pkg/sandboxpolicy/sections_test.go diff --git a/cmd/main.go b/cmd/main.go index 3886794e10..9af62780a3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -577,7 +577,8 @@ func main() { Kubernetes: k8sClient.GetKubernetesClient(), Dynamic: k8sClient.GetDynamicClient(), Containers: mainHandler.GetContainerCollection(), Tracers: mainHandler.GetTracerCollection(), NodeName: cfg.NodeName, AgentNamespace: cfg.NamespaceName, ImageName: cfg.SandboxPolicyImage, ImageArchive: "tracers.tar", - ContainerdSocket: cfg.SandboxPolicyContainerdSocket, ControlRoot: cfg.SandboxPolicyControlRoot, MaxOwners: cfg.SandboxPolicyMaxOwners, + PrivilegeImageName: cfg.SandboxPolicyPrivilegeImage, + ContainerdSocket: cfg.SandboxPolicyContainerdSocket, ControlRoot: cfg.SandboxPolicyControlRoot, MaxOwners: cfg.SandboxPolicyMaxOwners, Sources: sandboxintegration.NewProfileResolver(storageClient.GetStorageClient()), Receivers: thirdPartyTracers.ThirdPartyEventReceivers, OnError: func(err error) { logger.L().Ctx(ctx).Error("sandbox policy service diagnostic", helpers.Error(err)) }, diff --git a/pkg/config/config.go b/pkg/config/config.go index 473ecf268f..e03c52567c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -122,6 +122,7 @@ type Config struct { EnableMetricsExporter bool `mapstructure:"prometheusExporterEnabled"` EnableSandboxPolicy bool `mapstructure:"sandboxPolicyEnabled"` SandboxPolicyImage string `mapstructure:"sandboxPolicyImage"` + SandboxPolicyPrivilegeImage string `mapstructure:"sandboxPolicyPrivilegeImage"` SandboxPolicyContainerdSocket string `mapstructure:"sandboxPolicyContainerdSocket"` SandboxPolicyControlRoot string `mapstructure:"sandboxPolicyControlRoot"` SandboxPolicyMaxOwners uint64 `mapstructure:"sandboxPolicyMaxOwners"` @@ -226,6 +227,7 @@ func LoadConfigOptional(path string, errNotFound bool) (Config, error) { viper.SetDefault("sandboxPolicyEnabled", false) viper.SetDefault("sandboxPolicyImage", "") + viper.SetDefault("sandboxPolicyPrivilegeImage", "") viper.SetDefault("sandboxPolicyContainerdSocket", "") viper.SetDefault("sandboxPolicyControlRoot", "") viper.SetDefault("sandboxPolicyMaxOwners", 4) diff --git a/pkg/sandboxpolicy/compiler.go b/pkg/sandboxpolicy/compiler.go index cff260b1a2..1f69ffb23d 100644 --- a/pkg/sandboxpolicy/compiler.go +++ b/pkg/sandboxpolicy/compiler.go @@ -8,6 +8,7 @@ import ( "fmt" "sort" + privpolicy "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/policy" api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" ) @@ -30,8 +31,8 @@ func Compile(ctx context.Context, identity PolicyIdentity, spec api.PolicySpec, if spec.Mode != api.ModeAudit && spec.Mode != api.ModeEnforce { return result, fmt.Errorf("policy is not active: %q", spec.Mode) } - if spec.File != nil || spec.Network != nil || spec.Privilege != nil || spec.Identity != nil { - return result, fmt.Errorf("non-exec sections are unsupported") + if spec.File != nil || spec.Network != nil || spec.Identity != nil { + return result, fmt.Errorf("file, network and identity sections are unsupported") } // JSON cloning preserves all pointers and slice fields of the API vocabulary. data, err := json.Marshal(spec) @@ -49,6 +50,7 @@ func Compile(ctx context.Context, identity PolicyIdentity, spec api.PolicySpec, if copy.Exec != nil { result.Exec = *copy.Exec } + result.Privilege = copy.Privilege names := make(map[string]bool) for _, r := range result.Exec.Rules { names[r.Name] = true @@ -120,7 +122,7 @@ func Compile(ctx context.Context, identity PolicyIdentity, spec api.PolicySpec, } } // Validate imported rules as well as authored rules after class expansion. - validated := api.PolicySpec{Containers: copy.Containers, Mode: copy.Mode, RootRole: result.RootRole, Exec: &result.Exec} + validated := api.PolicySpec{Containers: copy.Containers, Mode: copy.Mode, RootRole: result.RootRole, Exec: &result.Exec, Privilege: result.Privilege} if err := api.ValidatePolicySpec(validated); err != nil { return result, err } @@ -166,8 +168,13 @@ func validateShape(spec api.PolicySpec) error { if spec.Mode == api.ModeOff { return nil } - if spec.File != nil || spec.Network != nil || spec.Privilege != nil || spec.Identity != nil { - return fmt.Errorf("non-exec sections are unsupported") + if spec.File != nil || spec.Network != nil || spec.Identity != nil { + return fmt.Errorf("file, network and identity sections are unsupported") + } + if spec.Privilege != nil { + if _, err := privpolicy.FromAPI("", spec.Mode, *spec.Privilege); err != nil { + return fmt.Errorf("privilege: %w", err) + } } if spec.Exec != nil { if spec.Exec.Default != api.ActionAllow && spec.Exec.Default != api.ActionDeny { diff --git a/pkg/sandboxpolicy/controller.go b/pkg/sandboxpolicy/controller.go index e9736a7508..799d8687a5 100644 --- a/pkg/sandboxpolicy/controller.go +++ b/pkg/sandboxpolicy/controller.go @@ -168,6 +168,56 @@ func (c *Controller) Reconcile(ctx context.Context) error { c.lastError = c.reconcile(ctx) return c.lastError } + +// PrepareScheduled passes, for every container of the given pods (scheduled to +// this node and not finished), the privilege policies it will get to +// a backend that can prepare them. It is a hint and changes no binding state. +func (c *Controller) PrepareScheduled(ctx context.Context, containers []Binding) error { + c.mu.Lock() + preparer, ok := c.backend.(SlotPreparer) + if !ok || c.configError != nil { + c.mu.Unlock() + return nil + } + policies := c.privilegePolicies() + c.mu.Unlock() + var plans []Plan + for _, b := range containers { + var plan Plan + for _, p := range policies { + if p.matches(b) { + plan.Policies = append(plan.Policies, CompiledPolicy{Identity: p.identity, Mode: p.spec.Mode, Privilege: p.spec.Privilege.DeepCopy()}) + } + } + if len(plan.Policies) > 0 { + plans = append(plans, plan) + } + } + // Outside c.mu: a preparation may start a gadget instance. + return preparer.Prepare(ctx, plans) +} + +// HasPrivilegePolicies reports whether any active policy has a privilege section. +func (c *Controller) HasPrivilegePolicies() bool { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.privilegePolicies()) > 0 +} + +// privilegePolicies is called with c.mu held. +func (c *Controller) privilegePolicies() []desiredPolicy { + var policies []desiredPolicy + for _, obj := range c.objects { + if obj.GetKind() == "SandboxPolicyClasses" { + continue + } + if p := parsePolicy(obj); p.err == nil && p.spec.Privilege != nil && p.spec.Mode != api.ModeOff { + policies = append(policies, p) + } + } + return policies +} + func (c *Controller) LastError() error { c.mu.Lock(); defer c.mu.Unlock(); return c.lastError } func bindingKey(b Binding) string { diff --git a/pkg/sandboxpolicy/igbackend/privilege.go b/pkg/sandboxpolicy/igbackend/privilege.go new file mode 100644 index 0000000000..cbcd790d86 --- /dev/null +++ b/pkg/sandboxpolicy/igbackend/privilege.go @@ -0,0 +1,360 @@ +package igbackend + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + "sync" + + "github.com/kubescape/node-agent/pkg/ebpf/gadgets/common/enforcement" + privpolicy "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/policy" + privruntime "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/runtime" + "github.com/kubescape/node-agent/pkg/sandboxpolicy" +) + +// PrivilegeEngine starts one node-wide enforce_privilege instance. Start returns +// only after native readiness, or after failed-candidate cleanup. +type PrivilegeEngine interface { + Start(ctx context.Context, id string, config privruntime.Config) (PrivilegeInstance, error) +} + +type PrivilegeInstance interface { + Ready() error + Bind(enforcement.Binding) error + Unbind(enforcement.ContainerIdentity) error + Stop(context.Context) error +} + +// PrivilegeStats counts how container binds found their policy slot. +type PrivilegeStats struct { + // Prepared: the slot already existed (created when the pod was scheduled, + // or used by another container), so the bind was a live map write. + Prepared uint64 + // Replaced: the slot was missing, so the node instance was replaced. + Replaced uint64 + // Replacements counts every node instance replacement. + Replacements uint64 +} + +type privilegeBinding struct { + binding sandboxpolicy.Binding + revision string + combination string + // lease pins the live container identity while it is bound (nil when the + // backend has no validator). + lease Lease +} + +func (p *privilegeBinding) check() error { + if p.lease == nil { + return nil + } + return p.lease.Check() +} + +func (p *privilegeBinding) close() error { + if p.lease == nil { + return nil + } + return p.lease.Close() +} + +// PrivilegeBackend owns ONE enforce_privilege instance per node. +// +// A slot is one combination of privilege policies; containers with the same +// combination share it. Binding or unbinding a container is a live map write. +// A combination that the running instance does not have (a new or changed +// policy) replaces the node instance: start the candidate with every slot in +// use, wait until it is ready with every container bound, then stop the old +// instance. Both enforce during the overlap. Unused slots are dropped at the +// next replacement. +type PrivilegeBackend struct { + mu sync.Mutex + engine PrivilegeEngine + validator Validator + prefix string + generation uint64 + instance PrivilegeInstance + slots map[string]uint32 // combination -> slot in instance + combinations map[string][]sandboxpolicy.CompiledPolicy + bindings map[identity]*privilegeBinding + prepared map[string][]sandboxpolicy.CompiledPolicy + retiring []PrivilegeInstance + stats PrivilegeStats + closed bool +} + +var ( + _ sandboxpolicy.Backend = (*PrivilegeBackend)(nil) + _ sandboxpolicy.ProtectionAttester = (*PrivilegeBackend)(nil) +) + +// NewPrivilege creates the node backend. validator verifies the live container +// identity before a bind and holds it while bound; nil skips the check (tests). +func NewPrivilege(engine PrivilegeEngine, validator Validator, prefix string) (*PrivilegeBackend, error) { + if engine == nil { + return nil, fmt.Errorf("privilege engine is required") + } + return &PrivilegeBackend{engine: engine, validator: validator, prefix: prefix, slots: map[string]uint32{}, combinations: map[string][]sandboxpolicy.CompiledPolicy{}, bindings: map[identity]*privilegeBinding{}, prepared: map[string][]sandboxpolicy.CompiledPolicy{}}, nil +} + +// Combination returns the privilege policies of a plan, sorted, and their key. +func Combination(plan sandboxpolicy.Plan) (string, []sandboxpolicy.CompiledPolicy) { + var policies []sandboxpolicy.CompiledPolicy + for _, p := range plan.Policies { + if p.Privilege != nil { + policies = append(policies, p) + } + } + sort.Slice(policies, func(i, j int) bool { return policies[i].Identity.UID < policies[j].Identity.UID }) + type part struct { + UID, Name, Mode string + Privilege any + } + parts := make([]part, len(policies)) + for i, p := range policies { + parts[i] = part{string(p.Identity.UID), p.Identity.Namespace + "/" + p.Identity.Name, string(p.Mode), p.Privilege} + } + data, _ := json.Marshal(parts) // concrete JSON-serializable API structs + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), policies +} + +func (b *PrivilegeBackend) Stats() PrivilegeStats { + b.mu.Lock() + defer b.mu.Unlock() + return b.stats +} + +func (b *PrivilegeBackend) Replace(ctx context.Context, binding sandboxpolicy.Binding, plan sandboxpolicy.Plan) (sandboxpolicy.ReplaceResult, error) { + b.mu.Lock() + defer b.mu.Unlock() + k := key(binding) + current := b.bindings[k] + var result sandboxpolicy.ReplaceResult + var health error + if current != nil { + result.ActiveRevision = current.revision + health = errors.Join(b.instance.Ready(), current.check()) + result.ProtectionUnconfirmed = health != nil + } + if b.closed { + return result, fmt.Errorf("privilege backend is closed") + } + combination, policies := Combination(plan) + if len(policies) == 0 { + return result, fmt.Errorf("plan has no privilege section") + } + if b.instance != nil && health == nil { + health = b.instance.Ready() + } + if current != nil && current.revision == plan.Revision && current.combination == combination && health == nil { + result.Committed = true + return result, b.retire(ctx) + } + next := &privilegeBinding{binding: binding, revision: plan.Revision, combination: combination} + if current != nil && current.lease != nil && current.check() == nil { + next.lease = current.lease + } else if b.validator != nil { + lease, err := b.validator.Acquire(ctx, binding) + if err != nil { + return result, err + } + next.lease = lease + } + commit := func() (sandboxpolicy.ReplaceResult, error) { + var errs []error + if current != nil && current.lease != nil && current.lease != next.lease { + errs = append(errs, current.close()) + } + errs = append(errs, b.retire(ctx)) + return sandboxpolicy.ReplaceResult{Committed: true, ActiveRevision: plan.Revision}, errors.Join(errs...) + } + fail := func(err error) (sandboxpolicy.ReplaceResult, error) { + if next.lease != nil && (current == nil || next.lease != current.lease) { + err = errors.Join(err, next.close()) + } + return result, err + } + if slot, ok := b.slots[combination]; ok && b.instance != nil && health == nil { + if err := b.instance.Bind(enforcement.Binding{Container: native(binding), Slot: slot}); err != nil { + return fail(err) + } + if current == nil { + b.stats.Prepared++ + } + b.bindings[k] = next + return commit() + } + extra := map[string][]sandboxpolicy.CompiledPolicy{combination: policies} + err := b.replaceInstance(ctx, b.prepared, extra, k, next) + if err != nil && len(b.prepared) > 0 { + // The prepared set may not fit; never let it block a real container. + err = b.replaceInstance(ctx, nil, extra, k, next) + } + if err != nil { + return fail(err) + } + if current == nil { + b.stats.Replaced++ + } + return commit() +} + +func (b *PrivilegeBackend) Withdraw(ctx context.Context, binding sandboxpolicy.Binding) (sandboxpolicy.WithdrawResult, error) { + b.mu.Lock() + defer b.mu.Unlock() + k := key(binding) + current := b.bindings[k] + if current == nil { + return sandboxpolicy.WithdrawResult{Committed: true}, b.retire(ctx) + } + // An instance that is not ready protects nothing, so the removal is + // already true; only a ready instance must drop the map entry first. + if b.instance != nil && b.instance.Ready() == nil { + if err := b.instance.Unbind(native(binding)); err != nil { + return sandboxpolicy.WithdrawResult{ProtectionUnconfirmed: true}, err + } + } + delete(b.bindings, k) + return sandboxpolicy.WithdrawResult{Committed: true}, errors.Join(current.close(), b.retire(ctx)) +} + +func (b *PrivilegeBackend) Attest(_ context.Context, binding sandboxpolicy.Binding, revision string) error { + b.mu.Lock() + defer b.mu.Unlock() + current := b.bindings[key(binding)] + if current == nil || current.revision != revision || b.instance == nil { + return fmt.Errorf("privilege revision %q is not active", revision) + } + return errors.Join(b.instance.Ready(), current.check()) +} + +// Prepare declares the combinations of the pods scheduled to this node that are +// not finished, so containers that start later find their slot. A missing combination replaces the node +// instance now, so the containers later bind with a live map write. +// Combinations no longer prepared and not in use are dropped at the next +// replacement. +func (b *PrivilegeBackend) Prepare(ctx context.Context, plans []sandboxpolicy.Plan) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return fmt.Errorf("privilege backend is closed") + } + prepared := map[string][]sandboxpolicy.CompiledPolicy{} + missing := false + for _, plan := range plans { + combination, policies := Combination(plan) + if len(policies) == 0 { + continue + } + prepared[combination] = policies + if _, ok := b.slots[combination]; !ok { + missing = true + } + } + if !missing { + b.prepared = prepared + return b.retire(ctx) + } + // The prepared set is kept only when the instance with it is running. + return errors.Join(b.replaceInstance(ctx, prepared, nil, identity{}, nil), b.retire(ctx)) +} + +// replaceInstance starts a candidate with every combination in use or prepared +// (plus extra), binds every container (plus next, under nextKey), and only then +// stops the old instance. On failure the old instance and state stay. +func (b *PrivilegeBackend) replaceInstance(ctx context.Context, prepared, extra map[string][]sandboxpolicy.CompiledPolicy, nextKey identity, next *privilegeBinding) error { + bindings := map[identity]*privilegeBinding{} + for k, v := range b.bindings { + bindings[k] = v + } + if next != nil { + bindings[nextKey] = next + } + wanted := map[string][]sandboxpolicy.CompiledPolicy{} + for c, p := range prepared { + wanted[c] = p + } + for c, p := range extra { + wanted[c] = p + } + for _, v := range bindings { + if _, ok := wanted[v.combination]; !ok { + wanted[v.combination] = b.combinations[v.combination] + } + } + keys := make([]string, 0, len(wanted)) + for c := range wanted { + keys = append(keys, c) + } + sort.Strings(keys) + slots := map[string]uint32{} + docs := make([][]privpolicy.Document, len(keys)) + for i, c := range keys { + slots[c] = uint32(i) + for _, p := range wanted[c] { + doc, err := privpolicy.FromAPI(p.Identity.Namespace+"/"+p.Identity.Name, p.Mode, *p.Privilege) + if err != nil { + return err + } + docs[i] = append(docs[i], doc) + } + } + compiled, err := privpolicy.CompileSlots(docs, b.generation+1) + if err != nil { + return err + } + config := privruntime.Config{Policy: compiled} + for _, v := range bindings { + config.InitialBindings = append(config.InitialBindings, enforcement.Binding{Container: native(v.binding), Slot: slots[v.combination]}) + } + b.generation++ + candidate, err := b.engine.Start(ctx, fmt.Sprintf("%s-privilege-%d", b.prefix, b.generation), config) + if err != nil { + return err + } + if err = candidate.Ready(); err != nil { + return errors.Join(err, candidate.Stop(ctx)) + } + if b.instance != nil { + b.retiring = append(b.retiring, b.instance) + } + b.instance, b.slots, b.combinations, b.bindings, b.prepared = candidate, slots, wanted, bindings, prepared + b.stats.Replacements++ + return nil +} + +// retire stops replaced instances and keeps the ones that still fail. +func (b *PrivilegeBackend) retire(ctx context.Context) error { + var kept []PrivilegeInstance + var errs []error + for _, i := range b.retiring { + if err := i.Stop(ctx); err != nil { + kept = append(kept, i) + errs = append(errs, err) + } + } + b.retiring = kept + return errors.Join(errs...) +} + +func (b *PrivilegeBackend) Close(ctx context.Context) error { + b.mu.Lock() + defer b.mu.Unlock() + b.closed = true + if b.instance != nil { + b.retiring = append(b.retiring, b.instance) + b.instance = nil + } + var errs []error + for _, v := range b.bindings { + errs = append(errs, v.close()) + } + b.bindings = map[identity]*privilegeBinding{} + return errors.Join(append(errs, b.retire(ctx))...) +} diff --git a/pkg/sandboxpolicy/igbackend/privilege_kernel_test.go b/pkg/sandboxpolicy/igbackend/privilege_kernel_test.go new file mode 100644 index 0000000000..3f552cb629 --- /dev/null +++ b/pkg/sandboxpolicy/igbackend/privilege_kernel_test.go @@ -0,0 +1,134 @@ +//go:build linux && sandboxpolicy_kernel + +package igbackend + +// Opt-in kernel test of the privilege backend with the real gadget image. +// Requires root, BPF LSM and a disposable Docker container started with +// seccomp=unconfined, apparmor=unconfined and --cap-add SYS_ADMIN: +// +// IG_PRIVILEGE_KERNEL_TEST=1 +// IG_PRIVILEGE_IMAGE_ARCHIVE OCI archive from `ig image export` +// IG_PRIVILEGE_IMAGE image name in the archive +// IG_PRIVILEGE_CONTAINER_ID full Docker container ID +// IG_PRIVILEGE_CGROUP_PATH its cgroup v2 path (relative to /sys/fs/cgroup) +// IG_PRIVILEGE_CONTAINER_ID2, IG_PRIVILEGE_CGROUP_PATH2 a second container +// +// With two containers it also checks the node-wide instance: the second +// container with the same policy is a live bind (no new instance), and a +// second container with another policy replaces the instance while the first +// container stays protected. + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/inspektor-gadget/inspektor-gadget/pkg/operators/simple" + "github.com/inspektor-gadget/inspektor-gadget/pkg/runtime/local" + "github.com/kubescape/node-agent/pkg/sandboxpolicy" + api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" + "golang.org/x/sys/unix" + orasoci "oras.land/oras-go/v2/content/oci" +) + +func TestPrivilegeBackendKernel(t *testing.T) { + if os.Getenv("IG_PRIVILEGE_KERNEL_TEST") != "1" { + t.Skip("explicit disposable kernel fixture opt-in required") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + bindingOf := func(id, cgroupPath string) sandboxpolicy.Binding { + var st unix.Stat_t + if err := unix.Stat(filepath.Join("/sys/fs/cgroup", cgroupPath), &st); err != nil { + t.Fatal(err) + } + return sandboxpolicy.Binding{Runtime: "docker", ContainerID: id, CgroupID: st.Ino, CgroupPath: cgroupPath} + } + id := os.Getenv("IG_PRIVILEGE_CONTAINER_ID") + binding := bindingOf(id, os.Getenv("IG_PRIVILEGE_CGROUP_PATH")) + + store, err := orasoci.NewFromTar(ctx, os.Getenv("IG_PRIVILEGE_IMAGE_ARCHIVE")) + if err != nil { + t.Fatal(err) + } + rt := local.New() + if err = rt.Init(nil); err != nil { + t.Fatal(err) + } + defer rt.Close() + engine, err := NewNativePrivilegeEngine(rt, simple.New("no-kubernetes"), store, os.Getenv("IG_PRIVILEGE_IMAGE")) + if err != nil { + t.Fatal(err) + } + backend, err := NewPrivilege(engine, nil, "kernel-test") + if err != nil { + t.Fatal(err) + } + defer backend.Close(context.Background()) + + unshareIn := func(id string) error { + return exec.CommandContext(ctx, "docker", "exec", id, "unshare", "-m", "true").Run() + } + unshare := func() error { return unshareIn(id) } + plan := func(revision string, action api.Action) sandboxpolicy.Plan { + return sandboxpolicy.Plan{Revision: revision, Policies: []sandboxpolicy.CompiledPolicy{{Mode: api.ModeEnforce, Privilege: &api.PrivilegeSection{ + Default: api.ActionAllow, + Rules: []api.PrivilegeRule{{Rule: api.Rule{Name: "mount-namespace", Action: action}, + Target: api.PrivilegeTarget{Syscall: []string{"unshare"}, Flags: []string{"CLONE_NEWNS"}}}}, + }}}} + } + + if err = unshare(); err != nil { + t.Fatalf("baseline unshare failed: %v", err) + } + if r, err := backend.Replace(ctx, binding, plan("deny", api.ActionDeny)); err != nil || !r.Committed { + t.Fatalf("deny plan: %+v %v", r, err) + } + if err = backend.Attest(ctx, binding, "deny"); err != nil { + t.Fatal(err) + } + if unshare() == nil { + t.Fatal("unshare allowed under the deny plan") + } + if id2 := os.Getenv("IG_PRIVILEGE_CONTAINER_ID2"); id2 != "" { + second := bindingOf(id2, os.Getenv("IG_PRIVILEGE_CGROUP_PATH2")) + if r, err := backend.Replace(ctx, second, plan("deny", api.ActionDeny)); err != nil || !r.Committed { + t.Fatalf("second container, same policy: %+v %v", r, err) + } + if s := backend.Stats(); s.Prepared != 1 || s.Replacements != 1 { + t.Fatalf("same policy must be a live bind: %+v", s) + } + if unshareIn(id2) == nil { + t.Fatal("second container not denied") + } + other := plan("other", api.ActionAllow) + other.Policies[0].Identity.UID = "other" + if r, err := backend.Replace(ctx, second, other); err != nil || !r.Committed { + t.Fatalf("second container, other policy: %+v %v", r, err) + } + if s := backend.Stats(); s.Replacements != 2 { + t.Fatalf("a new combination must replace the instance: %+v", s) + } + if unshareIn(id2) != nil { + t.Fatal("second container must be allowed by its new policy") + } + if unshare() == nil { + t.Fatal("first container lost protection across the replacement") + } + if r, err := backend.Withdraw(ctx, second); err != nil || !r.Committed { + t.Fatalf("withdraw second: %+v %v", r, err) + } + } + if r, err := backend.Replace(ctx, binding, plan("allow", api.ActionAllow)); err != nil || !r.Committed { + t.Fatalf("allow plan: %+v %v", r, err) + } + if err = unshare(); err != nil { + t.Fatalf("unshare denied under the allow plan: %v", err) + } + if r, err := backend.Withdraw(ctx, binding); err != nil || !r.Committed { + t.Fatalf("withdraw: %+v %v", r, err) + } +} diff --git a/pkg/sandboxpolicy/igbackend/privilege_native.go b/pkg/sandboxpolicy/igbackend/privilege_native.go new file mode 100644 index 0000000000..86a20b8ac5 --- /dev/null +++ b/pkg/sandboxpolicy/igbackend/privilege_native.go @@ -0,0 +1,140 @@ +package igbackend + +import ( + "context" + "errors" + "fmt" + "strconv" + + "github.com/inspektor-gadget/inspektor-gadget/pkg/datasource" + gadgetcontext "github.com/inspektor-gadget/inspektor-gadget/pkg/gadget-context" + "github.com/inspektor-gadget/inspektor-gadget/pkg/gadget-service/api" + apihelpers "github.com/inspektor-gadget/inspektor-gadget/pkg/gadget-service/api-helpers" + instancemanager "github.com/inspektor-gadget/inspektor-gadget/pkg/gadget-service/instance-manager" + "github.com/inspektor-gadget/inspektor-gadget/pkg/operators" + ocihandler "github.com/inspektor-gadget/inspektor-gadget/pkg/operators/oci-handler" + "github.com/inspektor-gadget/inspektor-gadget/pkg/operators/simple" + "github.com/inspektor-gadget/inspektor-gadget/pkg/runtime" + "github.com/kubescape/go-logger" + "github.com/kubescape/go-logger/helpers" + "github.com/kubescape/node-agent/pkg/ebpf/gadgets/common/enforcement" + privruntime "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/runtime" + "oras.land/oras-go/v2" +) + +// NewNativePrivilegeEngine uses an already initialized runtime, like NewNative. +// Each Start is one node-wide instance. +// Privilege verdicts are logged; exporting them as events is not wired yet. +func NewNativePrivilegeEngine(rt runtime.Runtime, kube operators.DataOperator, store oras.ReadOnlyTarget, image string) (PrivilegeEngine, error) { + if rt == nil || kube == nil || store == nil || image == "" { + return nil, fmt.Errorf("runtime, Kubernetes operator, OCI store and privilege image are required") + } + manager, err := instancemanager.New(rt) + if err != nil { + return nil, err + } + return &nativePrivilegeEngine{manager: manager, kube: kube, store: store, image: image}, nil +} + +type nativePrivilegeEngine struct { + manager *instancemanager.Manager + kube operators.DataOperator + store oras.ReadOnlyTarget + image string +} + +type nativePrivilegeInstance struct { + manager *instancemanager.Manager + id string + bindings *enforcement.Bindings + revision uint64 +} + +func (i *nativePrivilegeInstance) Ready() error { + status := i.bindings.Status() + if !status.Ready || status.Error != nil || status.Revision != i.revision { + return fmt.Errorf("privilege instance not ready: %+v", status) + } + return nil +} + +func (i *nativePrivilegeInstance) Bind(b enforcement.Binding) error { return i.bindings.Bind(b) } + +func (i *nativePrivilegeInstance) Unbind(identity enforcement.ContainerIdentity) error { + return i.bindings.Unbind(identity) +} + +func (i *nativePrivilegeInstance) Stop(ctx context.Context) error { + err := i.manager.StopGadget(ctx, i.id) + if errors.Is(err, instancemanager.ErrNotFound) { + return nil + } + return err +} + +func (e *nativePrivilegeEngine) Start(ctx context.Context, id string, config privruntime.Config) (PrivilegeInstance, error) { + handler := ocihandler.New() + params := apihelpers.ToParamDescs(handler.GlobalParams()).ToParams() + if err := params.Set("allow-enforcement", "true"); err != nil { + return nil, err + } + // Same trust model as NewNative: the archive is administrator-supplied. + if err := handler.Init(params); err != nil { + return nil, err + } + instance := &nativePrivilegeInstance{manager: e.manager, id: id, revision: config.Policy.Generation} + names := config.Policy.RuleNames + events := simple.New("sandbox-privilege-verdict", simple.WithPriority(100), simple.OnInit(func(gc operators.GadgetContext) error { + ds := gc.GetDataSources()["verdict"] + if ds == nil { + return fmt.Errorf("missing privilege verdict datasource") + } + var err error + instance.bindings, err = privruntime.GetBindings(gc) + if err != nil { + return err + } + fields := map[string]datasource.FieldAccessor{} + for _, name := range []string{"generation", "cgroup", "deny", "mode", "host_pid", "rule_id", "comm", "hook", "syscall", "capability", "fstype"} { + if fields[name] = ds.GetField(name); fields[name] == nil { + return fmt.Errorf("missing verdict field %q", name) + } + } + return ds.Subscribe(func(ds datasource.DataSource, data datasource.Data) error { + ruleID, _ := fields["rule_id"].Uint32(data) + rule := "" + if int(ruleID) < len(names) { + rule = names[ruleID] + } + mode, _ := fields["mode"].Uint32(data) + pid, _ := fields["host_pid"].Uint32(data) + cgroup, _ := fields["cgroup"].Uint64(data) + generation, _ := fields["generation"].Uint64(data) + capability, _ := fields["capability"].Int32(data) + comm, _ := fields["comm"].String(data) + hook, _ := fields["hook"].String(data) + syscall, _ := fields["syscall"].String(data) + fstype, _ := fields["fstype"].String(data) + logger.L().Info("sandbox privilege verdict", + helpers.String("rule", rule), helpers.Int("mode", int(mode)), helpers.Int("pid", int(pid)), + helpers.String("cgroup", fmt.Sprint(cgroup)), helpers.String("generation", fmt.Sprint(generation)), + helpers.String("comm", comm), helpers.String("hook", hook), helpers.String("syscall", syscall), + helpers.Int("capability", int(capability)), helpers.String("fstype", fstype)) + return nil // Reporting never changes enforcement. + }, 100) + })) + runtimeMntns, err := enforcement.RuntimeMountNamespace() + if err != nil { + return nil, fmt.Errorf("runtime mount namespace: %w", err) + } + desired := &api.GadgetInstance{Id: id, GadgetConfig: &api.GadgetRunRequest{Version: api.VersionGadgetRunProtocol, ImageName: e.image, + ParamValues: map[string]string{"operator.ebpf.host-mntns-id": strconv.FormatUint(runtimeMntns, 10)}}} + err = e.manager.StartGadget(ctx, desired, + gadgetcontext.WithOrasReadonlyTarget(e.store), + gadgetcontext.WithAdditionalDataOperators(handler, e.kube, privruntime.New(), events), + privruntime.WithConfig(config)) + if err != nil { + return nil, err + } + return instance, nil +} diff --git a/pkg/sandboxpolicy/igbackend/privilege_test.go b/pkg/sandboxpolicy/igbackend/privilege_test.go new file mode 100644 index 0000000000..171a976fea --- /dev/null +++ b/pkg/sandboxpolicy/igbackend/privilege_test.go @@ -0,0 +1,216 @@ +package igbackend + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/kubescape/node-agent/pkg/ebpf/gadgets/common/enforcement" + privruntime "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/runtime" + "github.com/kubescape/node-agent/pkg/sandboxpolicy" + api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" + "k8s.io/apimachinery/pkg/types" +) + +type fakePrivilegeInstance struct { + readyErr, stopErr error + stops int + bound map[uint64]uint32 +} + +func (i *fakePrivilegeInstance) Ready() error { return i.readyErr } +func (i *fakePrivilegeInstance) Bind(b enforcement.Binding) error { + i.bound[b.Container.CgroupID] = b.Slot + return nil +} +func (i *fakePrivilegeInstance) Unbind(c enforcement.ContainerIdentity) error { + delete(i.bound, c.CgroupID) + return nil +} +func (i *fakePrivilegeInstance) Stop(context.Context) error { i.stops++; return i.stopErr } + +type fakePrivilegeEngine struct { + instances []*fakePrivilegeInstance + configs []privruntime.Config + startErr error +} + +func (e *fakePrivilegeEngine) Start(_ context.Context, _ string, c privruntime.Config) (PrivilegeInstance, error) { + if e.startErr != nil { + return nil, e.startErr + } + i := &fakePrivilegeInstance{bound: map[uint64]uint32{}} + for _, b := range c.InitialBindings { + i.bound[b.Container.CgroupID] = b.Slot + } + e.instances = append(e.instances, i) + e.configs = append(e.configs, c) + return i, nil +} + +func privBinding(cgroup uint64) sandboxpolicy.Binding { + return sandboxpolicy.Binding{Runtime: "containerd", ContainerID: string(rune('a' + cgroup)), CgroupID: cgroup, CgroupPath: "/kubepods/c"} +} + +func privPolicy(uid string, def api.Action, rules ...api.PrivilegeRule) sandboxpolicy.CompiledPolicy { + return sandboxpolicy.CompiledPolicy{Identity: sandboxpolicy.PolicyIdentity{UID: types.UID(uid), Namespace: "ns", Name: uid}, Mode: api.ModeEnforce, Privilege: &api.PrivilegeSection{Default: def, Rules: rules}} +} + +func privPlan(revision string, policies ...sandboxpolicy.CompiledPolicy) sandboxpolicy.Plan { + return sandboxpolicy.Plan{Revision: revision, Policies: policies} +} + +var denyMount = api.PrivilegeRule{Rule: api.Rule{Name: "no-mount", Action: api.ActionDeny, Origin: "o"}, Target: api.PrivilegeTarget{Syscall: []string{"mount"}}} +var allowMount = api.PrivilegeRule{Rule: api.Rule{Name: "mount-ok", Action: api.ActionAllow}, Target: api.PrivilegeTarget{Syscall: []string{"mount"}}} + +func TestPrivilegeNodeInstanceSharesSlots(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + plan := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + if r, err := b.Replace(ctx, privBinding(1), plan); err != nil || !r.Committed { + t.Fatalf("first: %+v %v", r, err) + } + if r, err := b.Replace(ctx, privBinding(2), plan); err != nil || !r.Committed { + t.Fatalf("second: %+v %v", r, err) + } + if len(e.instances) != 1 || e.instances[0].bound[2] != e.instances[0].bound[1] { + t.Fatalf("same combination must share one slot in one instance: %d %+v", len(e.instances), e.instances[0].bound) + } + if s := b.Stats(); s.Replaced != 1 || s.Prepared != 1 { + t.Fatalf("stats %+v", s) + } + // Same plan again: attestation, no new instance. + if r, err := b.Replace(ctx, privBinding(1), plan); err != nil || !r.Committed || len(e.instances) != 1 { + t.Fatalf("same plan: %+v %v", r, err) + } + if err := b.Attest(ctx, privBinding(1), "r1"); err != nil { + t.Fatal(err) + } + // Withdraw is a live unbind. + if r, err := b.Withdraw(ctx, privBinding(2)); err != nil || !r.Committed { + t.Fatalf("withdraw: %+v %v", r, err) + } + if _, ok := e.instances[0].bound[2]; ok || len(e.instances) != 1 { + t.Fatal("withdraw must unbind in the running instance") + } +} + +func TestPrivilegeNewCombinationReplacesWithOverlap(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + if _, err := b.Replace(ctx, privBinding(1), privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount))); err != nil { + t.Fatal(err) + } + if _, err := b.Replace(ctx, privBinding(2), privPlan("r2", privPolicy("p2", api.ActionDeny))); err != nil { + t.Fatal(err) + } + if len(e.instances) != 2 || e.instances[0].stops != 1 || e.instances[1].stops != 0 { + t.Fatal("the new instance starts before the old one stops") + } + if c := e.configs[1]; c.Policy.Slots != 2 || len(c.InitialBindings) != 2 || c.Policy.Generation <= e.configs[0].Policy.Generation { + t.Fatalf("candidate must carry every slot and binding: %+v", c) + } + // A failed candidate keeps the old instance and state. + e.startErr = errors.New("load failed") + if r, err := b.Replace(ctx, privBinding(3), privPlan("r3", privPolicy("p3", api.ActionDeny))); err == nil || r.Committed { + t.Fatalf("expected failure: %+v", r) + } + if e.instances[1].stops != 0 || b.Attest(ctx, privBinding(1), "r1") != nil { + t.Fatal("old instance must keep running with its bindings") + } +} + +func TestPrivilegePrepareCreatesSlotBeforeContainer(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + plan := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + if err := b.Prepare(ctx, []sandboxpolicy.Plan{plan}); err != nil { + t.Fatal(err) + } + if len(e.instances) != 1 || e.configs[0].Policy.Slots != 1 { + t.Fatalf("prepare must create the slot: %d", len(e.instances)) + } + if _, err := b.Replace(ctx, privBinding(1), plan); err != nil || len(e.instances) != 1 { + t.Fatalf("bind after prepare must be live: %v instances=%d", err, len(e.instances)) + } + if s := b.Stats(); s.Prepared != 1 || s.Replaced != 0 { + t.Fatalf("stats %+v", s) + } + // The same prepare again changes nothing. + if err := b.Prepare(ctx, []sandboxpolicy.Plan{plan}); err != nil || len(e.instances) != 1 { + t.Fatalf("repeat prepare: %v", err) + } +} + +func TestPrivilegeMergesPoliciesInOneSlot(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + plan := privPlan("r1", privPolicy("p1", api.ActionDeny, denyMount), privPolicy("p2", api.ActionAllow, allowMount)) + if _, err := b.Replace(context.Background(), privBinding(1), plan); err != nil { + t.Fatal(err) + } + c := e.configs[0].Policy + names := strings.Join(c.RuleNames, ",") + if c.Slots != 1 || !strings.Contains(names, "ns/p1/no-mount") || !strings.Contains(names, "ns/p2/mount-ok") { + t.Fatalf("merged slot: slots=%d names=%s", c.Slots, names) + } +} + +func TestPrivilegeRetriesFailedStop(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + if _, err := b.Replace(ctx, privBinding(1), privPlan("r1", privPolicy("p1", api.ActionAllow))); err != nil { + t.Fatal(err) + } + e.instances[0].stopErr = errors.New("busy") + r, err := b.Replace(ctx, privBinding(1), privPlan("r2", privPolicy("p1", api.ActionDeny))) + if err == nil || !r.Committed || r.ActiveRevision != "r2" { + t.Fatalf("committed with cleanup error expected: %+v %v", r, err) + } + e.instances[0].stopErr = nil + if _, err = b.Replace(ctx, privBinding(1), privPlan("r2", privPolicy("p1", api.ActionDeny))); err != nil || e.instances[0].stops != 2 { + t.Fatalf("retry: %v stops=%d", err, e.instances[0].stops) + } +} + +func TestPrivilegeWithdrawCommitsWhenInstanceDown(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + if _, err := b.Replace(ctx, privBinding(1), privPlan("r1", privPolicy("p1", api.ActionAllow))); err != nil { + t.Fatal(err) + } + e.instances[0].readyErr = errors.New("stopped") + if r, err := b.Withdraw(ctx, privBinding(1)); err != nil || !r.Committed { + t.Fatalf("withdraw on a stopped instance must commit: %+v %v", r, err) + } + // The next container recovers the node with a new instance. + e.instances[0].readyErr = nil + if r, err := b.Replace(ctx, privBinding(2), privPlan("r2", privPolicy("p1", api.ActionAllow))); err != nil || !r.Committed { + t.Fatalf("recovery: %+v %v", r, err) + } + if c := e.configs[len(e.configs)-1]; len(c.InitialBindings) > 1 { + t.Fatalf("the withdrawn container must not be carried: %+v", c.InitialBindings) + } +} + +func TestPrivilegeOversizedPrepareDoesNotBlockContainers(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + var plans []sandboxpolicy.Plan + for i := 0; i < 65; i++ { // one more than the slots per node + plans = append(plans, privPlan("p", privPolicy(string(rune('A'+i)), api.ActionAllow))) + } + if err := b.Prepare(ctx, plans); err == nil { + t.Fatal("too many combinations must fail") + } + if r, err := b.Replace(ctx, privBinding(1), privPlan("r1", privPolicy("real", api.ActionDeny))); err != nil || !r.Committed { + t.Fatalf("a real container must still bind: %+v %v", r, err) + } +} diff --git a/pkg/sandboxpolicy/integration/bindings.go b/pkg/sandboxpolicy/integration/bindings.go index 8efcf4c8bb..59808445f5 100644 --- a/pkg/sandboxpolicy/integration/bindings.go +++ b/pkg/sandboxpolicy/integration/bindings.go @@ -61,6 +61,46 @@ func SnapshotSelectedBindingsFromContainers(ctx context.Context, client kubernet return snapshotBindings(ctx, client, containers, nodeName, selector) } +// ScheduledContainers returns API metadata (no runtime identity) for every +// declared regular and init container of the pods scheduled to this node that +// are not finished. It feeds privilege slot preparation, not binding. +func ScheduledContainers(ctx context.Context, client kubernetes.Interface, nodeName string) ([]sandboxpolicy.Binding, error) { + if client == nil || nodeName == "" { + return nil, fmt.Errorf("Kubernetes client and node name are required") + } + pods, err := client.CoreV1().Pods("").List(ctx, metav1.ListOptions{FieldSelector: fields.OneTermEqualSelector("spec.nodeName", nodeName).String()}) + if err != nil { + return nil, fmt.Errorf("list node pods: %w", err) + } + namespaces, err := client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("list namespaces: %w", err) + } + namespaceLabels := make(map[string]map[string]string, len(namespaces.Items)) + for _, ns := range namespaces.Items { + namespaceLabels[ns.Name] = ns.Labels + } + var result []sandboxpolicy.Binding + for _, pod := range pods.Items { + if pod.Spec.NodeName != nodeName || pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { + continue + } + labels, ok := namespaceLabels[pod.Namespace] + if !ok { + continue + } + for _, group := range []struct { + kind string + containers []corev1.Container + }{{"regular", pod.Spec.Containers}, {"init", pod.Spec.InitContainers}} { + for _, c := range group.containers { + result = append(result, sandboxpolicy.Binding{PodUID: pod.UID, Namespace: pod.Namespace, PodName: pod.Name, ContainerName: c.Name, Kind: group.kind, PodLabels: copyLabels(pod.Labels), NamespaceLabels: copyLabels(labels)}) + } + } + } + return result, nil +} + func snapshotBindings(ctx context.Context, client kubernetes.Interface, containers []*containercollection.Container, nodeName string, selector func(sandboxpolicy.Binding) bool) ([]sandboxpolicy.Binding, error) { if client == nil || nodeName == "" { diff --git a/pkg/sandboxpolicy/integration/service.go b/pkg/sandboxpolicy/integration/service.go index fd920f5ffd..507ef596a1 100644 --- a/pkg/sandboxpolicy/integration/service.go +++ b/pkg/sandboxpolicy/integration/service.go @@ -22,8 +22,10 @@ import ( "github.com/kubescape/node-agent/pkg/utils" "github.com/kubescape/node-agent/pkg/watcher" api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" orasoci "oras.land/oras-go/v2/content/oci" @@ -43,6 +45,8 @@ type ServiceOptions struct { Receivers *maps.SafeMap[utils.EventType, mapset.Set[containerwatcher.GenericEventReceiver]] Enricher containerwatcher.TaskBasedEnricher OnError func(error) + // PrivilegeImageName enables the privilege section. Empty rejects it. + PrivilegeImageName string } type serviceBackend interface { @@ -118,11 +122,29 @@ func NewService(ctx context.Context, options ServiceOptions) (*Service, error) { _ = s.closeRuntime() return nil, err } - s.backend, err = igbackend.NewFull(engine, source, s.exporter, s.report) + full, err := igbackend.NewFull(engine, source, s.exporter, s.report) if err != nil { _ = s.closeRuntime() return nil, err } + var privilege sandboxpolicy.Backend + if options.PrivilegeImageName != "" { + privilegeEngine, err := igbackend.NewNativePrivilegeEngine(r, kubemanager.NewKubeManager(options.Containers, options.Tracers), store, options.PrivilegeImageName) + if err == nil { + privilege, err = igbackend.NewPrivilege(privilegeEngine, igbackend.LinuxValidator{ProcRoot: host.HostProcFs, CgroupRoot: filepath.Join(host.HostRoot, "sys/fs/cgroup")}, options.NodeName) + } + if err != nil { + _ = full.Close(ctx) + _ = s.closeRuntime() + return nil, err + } + } + s.backend, err = sandboxpolicy.NewSectionBackend(full, privilege) + if err != nil { + _ = full.Close(ctx) + _ = s.closeRuntime() + return nil, err + } s.Controller, err = sandboxpolicy.NewController(options.Dynamic, s.backend, options.Sources, readinessWriter{inner: &sandboxpolicy.DynamicStatusWriter{Client: options.Dynamic}, observed: &s.inputEpoch, snapshot: &s.snapshotEpoch}, sandboxpolicy.Options{Node: api.NodeReference{Name: node.Name, UID: node.UID}, AgentNamespace: options.AgentNamespace}) if err != nil { _ = s.backend.Close(ctx) @@ -135,7 +157,16 @@ func NewService(ctx context.Context, options ServiceOptions) (*Service, error) { // Watch events are hints only. Generic watcher queues may deliver old updates // after a relist/deletion; applying their payload could resurrect removed policy. // Refresh always reads authoritative configuration and cannot revive queued UIDs. -func (s *Service) WatchResources() []watcher.WatchResource { return s.Controller.WatchResources() } +// WatchResources adds this node's pods, so a scheduled pod can get its privilege +// slot before its containers start. +func (s *Service) WatchResources() []watcher.WatchResource { + resources := s.Controller.WatchResources() + if s.options.PrivilegeImageName == "" { + return resources + } + pods := watcher.NewWatchResource(schema.GroupVersionResource{Version: "v1", Resource: "pods"}, metav1.ListOptions{FieldSelector: "spec.nodeName=" + s.options.NodeName}) + return append(resources, pods) +} func (s *Service) AddHandler(_ context.Context, obj runtime.Object) { s.configurationHint(obj) } func (s *Service) ModifyHandler(_ context.Context, obj runtime.Object) { s.configurationHint(obj) } func (s *Service) DeleteHandler(_ context.Context, obj runtime.Object) { s.configurationHint(obj) } @@ -143,6 +174,12 @@ func (s *Service) configurationHint(obj runtime.Object) { if obj == nil { return } + if pod, ok := obj.(*corev1.Pod); ok { + if s.options.PrivilegeImageName != "" && pod.Spec.NodeName == s.options.NodeName && pod.Status.Phase == corev1.PodPending { + s.ContainerCallback(containercollection.PubSubEvent{}) + } + return + } switch obj.GetObjectKind().GroupVersionKind().Kind { case "SandboxPolicy", "ClusterSandboxPolicy", "SandboxPolicyClasses": s.ContainerCallback(containercollection.PubSubEvent{}) @@ -206,6 +243,15 @@ drain: } func (s *Service) refresh(ctx context.Context) { s.snapshotEpoch.Store(s.inputEpoch.Load()) + if s.options.PrivilegeImageName != "" && s.options.Kubernetes != nil && s.Controller.HasPrivilegePolicies() { + scheduled, err := ScheduledContainers(ctx, s.options.Kubernetes, s.options.NodeName) + if err == nil { + err = s.Controller.PrepareScheduled(ctx, scheduled) + } + if err != nil { + s.report(fmt.Errorf("privilege slot preparation: %w", err)) + } + } // Configuration and selected-runtime discovery share one commit barrier. A // candidate cannot report Ready between its relist and a failed binding list. err := s.Controller.Refresh(ctx, func(ctx context.Context, selected func(sandboxpolicy.Binding) bool) ([]sandboxpolicy.Binding, error) { @@ -303,7 +349,11 @@ func (s *Service) Close(ctx context.Context) error { // from ExporterStats. Unavailable counts/errors mean totals are incomplete; // these evaluator counters do not cover early native-owner exec refusals. func (s *Service) KernelEventStats(ctx context.Context) (igbackend.KernelEventStats, error) { - source, ok := s.backend.(interface { + var backend any = s.backend + if sections, isSections := s.backend.(*sandboxpolicy.SectionBackend); isSections { + backend = sections.Exec() + } + source, ok := backend.(interface { KernelEventStats(context.Context) (igbackend.KernelEventStats, error) }) if !ok { diff --git a/pkg/sandboxpolicy/sections.go b/pkg/sandboxpolicy/sections.go new file mode 100644 index 0000000000..d73bbae149 --- /dev/null +++ b/pkg/sandboxpolicy/sections.go @@ -0,0 +1,255 @@ +package sandboxpolicy + +import ( + "context" + "errors" + "fmt" + "sort" + "sync" + + api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" +) + +// SectionBackend drives one backend per policy section. +// +// A binding whose old and new plans have no privilege section goes straight to +// the exec backend, unchanged. Otherwise the plan is split into an exec plan +// and a privilege plan, each with its own revision. A policy's generation is part +// of the exec revision, so an edit restarts exec only when exec and privilege +// share one policy object; sections in separate policies stay independent. +// +// Commit follows the Backend contract of each section: a section that returns +// Committed=true is committed, even with a cleanup error. If the second section +// does not commit, the first section returns to its previous plan. +type SectionBackend struct { + mu sync.Mutex + exec Backend + privilege Backend // nil: privilege sections are rejected + active map[string]sectionPlans +} + +type sectionPlans struct { + revision string + exec, privilege Plan + // privilegeCleanup: a privilege withdraw committed with a cleanup error; + // later calls retry it. + privilegeCleanup bool +} + +var ( + _ Backend = (*SectionBackend)(nil) + _ ProtectionAttester = (*SectionBackend)(nil) +) + +func NewSectionBackend(exec, privilege Backend) (*SectionBackend, error) { + if exec == nil { + return nil, fmt.Errorf("exec backend is required") + } + return &SectionBackend{exec: exec, privilege: privilege, active: map[string]sectionPlans{}}, nil +} + +// execOnly reports whether the plans need only the exec backend. +func (p sectionPlans) execOnly() bool { return p.privilege.Revision == "" && !p.privilegeCleanup } + +// Prepare passes the plans of pods scheduled to this node, whose containers have +// not finished (running ones included), to a privilege backend that can create +// their slots before new containers start. +func (s *SectionBackend) Prepare(ctx context.Context, plans []Plan) error { + preparer, ok := s.privilege.(SlotPreparer) + if !ok { + return nil + } + return preparer.Prepare(ctx, plans) +} + +// Exec returns the exec section backend. +func (s *SectionBackend) Exec() Backend { return s.exec } + +func split(plan Plan) (sectionPlans, error) { + s := sectionPlans{revision: plan.Revision} + type execPart struct { + Identity PolicyIdentity + Mode api.PolicyMode + RootRole string + Exec api.ExecSection + } + var execParts []execPart + for _, p := range plan.Policies { + if p.Privilege != nil { + s.privilege.Policies = append(s.privilege.Policies, p) + } + } + if len(s.privilege.Policies) == 0 { + s.exec = plan + return s, nil + } + // The privilege compiler reads each policy's identity, mode and section. + type privilegePart struct { + Identity PolicyIdentity + Mode api.PolicyMode + Privilege *api.PrivilegeSection + } + parts := make([]privilegePart, 0, len(s.privilege.Policies)) + for _, p := range s.privilege.Policies { + id := p.Identity + id.Generation = 0 // an edit of another section must not change this revision + parts = append(parts, privilegePart{id, p.Mode, p.Privilege}) + } + sort.Slice(parts, func(i, j int) bool { return parts[i].Identity.UID < parts[j].Identity.UID }) + s.privilege.Revision = digest(parts) + for _, p := range plan.Policies { + if p.Exec.Default == "" { + continue + } + e := p + e.Privilege = nil + s.exec.Policies = append(s.exec.Policies, e) + execParts = append(execParts, execPart{p.Identity, p.Mode, p.RootRole, p.Exec}) + } + if len(execParts) > 0 { + s.exec.Revision = digest(execParts) + } + return s, nil +} + +type sectionResult struct { + committed, unconfirmed bool + err error +} + +// apply moves one section from old to want. +func apply(ctx context.Context, backend Backend, binding Binding, old, want Plan) sectionResult { + switch { + case want.Revision == old.Revision: + if want.Revision == "" { + return sectionResult{committed: true} + } + fallthrough // same plan: retry cleanup and confirm protection + case want.Revision != "": + r, err := backend.Replace(ctx, binding, want) + committed := r.Committed && r.ActiveRevision == want.Revision + if !committed && err == nil { + err = fmt.Errorf("replace not committed") + } + return sectionResult{committed, r.ProtectionUnconfirmed, err} + default: + r, err := backend.Withdraw(ctx, binding) + if !r.Committed && err == nil { + err = fmt.Errorf("withdraw not committed") + } + return sectionResult{r.Committed, r.ProtectionUnconfirmed, err} + } +} + +func (s *SectionBackend) Replace(ctx context.Context, binding Binding, plan Plan) (ReplaceResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + k := bindingKey(binding) + old := s.active[k] + want, err := split(plan) + if err != nil { + return ReplaceResult{}, err + } + if old.execOnly() && want.execOnly() { + r, err := s.exec.Replace(ctx, binding, plan) + if r.Committed { + s.active[k] = want + } + return r, err + } + if s.privilege == nil { + return ReplaceResult{}, fmt.Errorf("privilege sections are unsupported: no privilege image configured") + } + + e := apply(ctx, s.exec, binding, old.exec, want.exec) + if !e.committed { + return ReplaceResult{ProtectionUnconfirmed: e.unconfirmed}, fmt.Errorf("exec section: %w", e.err) + } + oldPrivilege := old.privilege + if old.privilegeCleanup && oldPrivilege.Revision == "" { + oldPrivilege.Revision = "cleanup" // forces a privilege Withdraw retry + } + p := apply(ctx, s.privilege, binding, oldPrivilege, want.privilege) + if !p.committed { + // Return exec to its previous plan. + back := apply(ctx, s.exec, binding, want.exec, old.exec) + return ReplaceResult{ProtectionUnconfirmed: e.unconfirmed || p.unconfirmed || back.unconfirmed || !back.committed}, + errors.Join(fmt.Errorf("privilege section: %w", p.err), back.err) + } + want.privilegeCleanup = want.privilege.Revision == "" && p.err != nil + s.active[k] = want + return ReplaceResult{Committed: true, ActiveRevision: plan.Revision, ProtectionUnconfirmed: e.unconfirmed || p.unconfirmed}, errors.Join(e.err, p.err) +} + +func (s *SectionBackend) Withdraw(ctx context.Context, binding Binding) (WithdrawResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + k := bindingKey(binding) + old := s.active[k] + if old.execOnly() { + r, err := s.exec.Withdraw(ctx, binding) + if r.Committed { + delete(s.active, k) + } + return r, err + } + e, eerr := s.exec.Withdraw(ctx, binding) + p, perr := s.privilege.Withdraw(ctx, binding) + result := WithdrawResult{ + Committed: e.Committed && p.Committed, + // One section removed and the other not: the old protection is gone. + ProtectionUnconfirmed: e.ProtectionUnconfirmed || p.ProtectionUnconfirmed || e.Committed != p.Committed, + } + switch { + case result.Committed && perr != nil: + s.active[k] = sectionPlans{privilegeCleanup: true} + case result.Committed: + delete(s.active, k) + case e.Committed: + old.exec = Plan{} + s.active[k] = old + } + return result, errors.Join(eerr, perr) +} + +// Attest checks each section that is active for the binding. +func (s *SectionBackend) Attest(ctx context.Context, binding Binding, revision string) error { + s.mu.Lock() + plans, ok := s.active[bindingKey(binding)] + s.mu.Unlock() + if !ok || plans.execOnly() { + if a, isAttester := s.exec.(ProtectionAttester); isAttester { + return a.Attest(ctx, binding, revision) + } + return fmt.Errorf("exec backend cannot attest") + } + if plans.revision != revision { + return fmt.Errorf("revision %q is not active", revision) + } + var errs []error + for _, section := range []struct { + backend Backend + plan Plan + }{{s.exec, plans.exec}, {s.privilege, plans.privilege}} { + if section.plan.Revision == "" { + continue + } + a, isAttester := section.backend.(ProtectionAttester) + if !isAttester { + errs = append(errs, fmt.Errorf("backend cannot attest")) + continue + } + errs = append(errs, a.Attest(ctx, binding, section.plan.Revision)) + } + return errors.Join(errs...) +} + +func (s *SectionBackend) Close(ctx context.Context) error { + var errs []error + for _, backend := range []Backend{s.exec, s.privilege} { + if c, ok := backend.(interface{ Close(context.Context) error }); ok { + errs = append(errs, c.Close(ctx)) + } + } + return errors.Join(errs...) +} diff --git a/pkg/sandboxpolicy/sections_test.go b/pkg/sandboxpolicy/sections_test.go new file mode 100644 index 0000000000..33c407baa0 --- /dev/null +++ b/pkg/sandboxpolicy/sections_test.go @@ -0,0 +1,248 @@ +package sandboxpolicy + +import ( + "context" + "testing" + + api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" +) + +func privilegeSection(def api.Action) *api.PrivilegeSection { + return &api.PrivilegeSection{Default: def, Rules: []api.PrivilegeRule{{Rule: api.Rule{Name: "no-mount", Action: api.ActionDeny}, Target: api.PrivilegeTarget{Syscall: []string{"mount"}}}}} +} + +func sectionPlan(revision string, policies ...CompiledPolicy) Plan { + return Plan{Revision: revision, Policies: policies} +} + +func TestSectionBackendExecOnlyPlanIsUnchanged(t *testing.T) { + exec, privilege := &fakeBackend{plans: map[string]Plan{}}, &fakeBackend{plans: map[string]Plan{}} + s, err := NewSectionBackend(exec, privilege) + if err != nil { + t.Fatal(err) + } + plan := sectionPlan("aggregate", CompiledPolicy{Revision: "p1", Exec: api.ExecSection{Default: api.ActionDeny}}) + r, err := s.Replace(context.Background(), Binding{ContainerID: "c"}, plan) + if err != nil || !r.Committed || r.ActiveRevision != "aggregate" { + t.Fatalf("replace: %+v %v", r, err) + } + if got := exec.plans[bindingKey(Binding{ContainerID: "c"})]; got.Revision != "aggregate" { + t.Fatalf("exec got %+v", got) + } + if privilege.replacements+privilege.withdrawals != 0 { + t.Fatal("privilege backend must not be called for an exec-only plan") + } + if err := s.Attest(context.Background(), Binding{ContainerID: "c"}, "aggregate"); err != nil { + t.Fatal(err) + } +} + +func TestSectionBackendSplitsAndKeepsUnchangedSection(t *testing.T) { + exec, privilege := &fakeBackend{plans: map[string]Plan{}}, &fakeBackend{plans: map[string]Plan{}} + s, _ := NewSectionBackend(exec, privilege) + b := Binding{ContainerID: "c"} + execPolicy := CompiledPolicy{Revision: "e", Exec: api.ExecSection{Default: api.ActionDeny}} + privPolicy := CompiledPolicy{Revision: "p", Privilege: privilegeSection(api.ActionAllow)} + if _, err := s.Replace(context.Background(), b, sectionPlan("r1", execPolicy, privPolicy)); err != nil { + t.Fatal(err) + } + if len(exec.plans[bindingKey(b)].Policies) != 1 || exec.plans[bindingKey(b)].Policies[0].Privilege != nil { + t.Fatalf("exec plan %+v", exec.plans[bindingKey(b)]) + } + if len(privilege.plans[bindingKey(b)].Policies) != 1 { + t.Fatalf("privilege plan %+v", privilege.plans[bindingKey(b)]) + } + // An exec-only change keeps the privilege revision: no privilege restart. + execPolicy.Revision = "e2" + execPolicy.Exec.Default = api.ActionAllow + if _, err := s.Replace(context.Background(), b, sectionPlan("r2", execPolicy, privPolicy)); err != nil { + t.Fatal(err) + } + if exec.replacements != 2 || privilege.replacements != 1 { + t.Fatalf("exec %d privilege %d replacements", exec.replacements, privilege.replacements) + } + // Dropping the privilege policy withdraws only the privilege section. + if _, err := s.Replace(context.Background(), b, sectionPlan("r3", execPolicy)); err != nil { + t.Fatal(err) + } + if privilege.withdrawals != 1 || exec.withdrawals != 0 { + t.Fatalf("withdrawals exec %d privilege %d", exec.withdrawals, privilege.withdrawals) + } +} + +func TestSectionBackendPrivilegeFailureRestoresExec(t *testing.T) { + exec, privilege := &fakeBackend{plans: map[string]Plan{}}, &fakeBackend{plans: map[string]Plan{}, fail: true} + s, _ := NewSectionBackend(exec, privilege) + b := Binding{ContainerID: "c"} + execPolicy := CompiledPolicy{Revision: "e", Exec: api.ExecSection{Default: api.ActionDeny}} + r, err := s.Replace(context.Background(), b, sectionPlan("r1", execPolicy, CompiledPolicy{Revision: "p", Privilege: privilegeSection(api.ActionAllow)})) + if err == nil || r.Committed { + t.Fatalf("expected failure, got %+v", r) + } + if _, ok := exec.plans[bindingKey(b)]; ok || exec.withdrawals != 1 { + t.Fatalf("exec must return to its previous (empty) state: %+v withdrawals=%d", exec.plans, exec.withdrawals) + } +} + +func TestSectionBackendRejectsPrivilegeWithoutImage(t *testing.T) { + s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, nil) + if _, err := s.Replace(context.Background(), Binding{}, sectionPlan("r", CompiledPolicy{Privilege: privilegeSection(api.ActionDeny)})); err == nil { + t.Fatal("privilege must be rejected without a privilege backend") + } +} + +func TestCompileKeepsPrivilegeSection(t *testing.T) { + spec := execSpec() + spec.Privilege = privilegeSection(api.ActionAllow) + got, err := Compile(context.Background(), PolicyIdentity{}, spec, api.PolicyClasses{}, Binding{}, nil) + if err != nil || got.Privilege == nil || got.Privilege.Rules[0].Name != "no-mount" { + t.Fatalf("compile: %+v %v", got.Privilege, err) + } + spec.Privilege.Default = "maybe" + if _, err := Compile(context.Background(), PolicyIdentity{}, spec, api.PolicyClasses{}, Binding{}, nil); err == nil { + t.Fatal("invalid privilege default accepted") + } +} + +func TestSectionBackendKeepsCommittedCleanupErrors(t *testing.T) { + // Exec-only: the exec backend result passes through unchanged. + exec := &fakeBackend{plans: map[string]Plan{}, postCommit: true} + s, _ := NewSectionBackend(exec, &fakeBackend{plans: map[string]Plan{}}) + b := Binding{ContainerID: "c"} + r, err := s.Replace(context.Background(), b, sectionPlan("r1", CompiledPolicy{Exec: api.ExecSection{Default: api.ActionDeny}})) + if err == nil || !r.Committed || r.ActiveRevision != "r1" { + t.Fatalf("exec-only cleanup error must stay committed: %+v %v", r, err) + } + // With privilege: a committed section with a cleanup error does not roll back. + privilege := &fakeBackend{plans: map[string]Plan{}, postCommit: true} + s, _ = NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege) + r, err = s.Replace(context.Background(), b, sectionPlan("r2", CompiledPolicy{Privilege: privilegeSection(api.ActionAllow)})) + if err == nil || !r.Committed || r.ActiveRevision != "r2" { + t.Fatalf("privilege cleanup error must stay committed: %+v %v", r, err) + } +} + +func TestSectionBackendPrivilegeChangeKeepsExecRevision(t *testing.T) { + exec, privilege := &fakeBackend{plans: map[string]Plan{}}, &fakeBackend{plans: map[string]Plan{}} + s, _ := NewSectionBackend(exec, privilege) + b := Binding{ContainerID: "c"} + p := CompiledPolicy{Revision: "v1", Exec: api.ExecSection{Default: api.ActionDeny}, Privilege: privilegeSection(api.ActionAllow)} + if _, err := s.Replace(context.Background(), b, sectionPlan("r1", p)); err != nil { + t.Fatal(err) + } + p.Revision = "v2" // the compiler's policy revision changes with the privilege section + p.Privilege = privilegeSection(api.ActionDeny) + if _, err := s.Replace(context.Background(), b, sectionPlan("r2", p)); err != nil { + t.Fatal(err) + } + if exec.replacements != 1 || privilege.replacements != 2 { + t.Fatalf("exec %d privilege %d replacements", exec.replacements, privilege.replacements) + } +} + +func TestSectionBackendSendsAllPrivilegePolicies(t *testing.T) { + privilege := &fakeBackend{plans: map[string]Plan{}} + s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege) + b := Binding{ContainerID: "c"} + p1 := CompiledPolicy{Identity: PolicyIdentity{UID: "1"}, Privilege: privilegeSection(api.ActionAllow)} + p2 := CompiledPolicy{Identity: PolicyIdentity{UID: "2"}, Privilege: privilegeSection(api.ActionDeny)} + if _, err := s.Replace(context.Background(), b, sectionPlan("r", p1, p2)); err != nil { + t.Fatal(err) + } + if got := privilege.plans[bindingKey(b)]; len(got.Policies) != 2 { + t.Fatalf("privilege plan %+v", got) + } +} + +type preparingBackend struct { + fakeBackend + prepared []Plan +} + +func (p *preparingBackend) Prepare(_ context.Context, plans []Plan) error { + p.prepared = plans + return nil +} + +func TestSectionBackendPassesPrepare(t *testing.T) { + privilege := &preparingBackend{fakeBackend: fakeBackend{plans: map[string]Plan{}}} + s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege) + if err := s.Prepare(context.Background(), []Plan{sectionPlan("r", CompiledPolicy{Privilege: privilegeSection(api.ActionDeny)})}); err != nil || len(privilege.prepared) != 1 { + t.Fatalf("prepare: %v %v", err, privilege.prepared) + } +} + +func TestSectionBackendModeChangeReachesPrivilege(t *testing.T) { + privilege := &fakeBackend{plans: map[string]Plan{}} + s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege) + b := Binding{ContainerID: "c"} + p := CompiledPolicy{Mode: api.ModeAudit, Privilege: privilegeSection(api.ActionAllow)} + if _, err := s.Replace(context.Background(), b, sectionPlan("r1", p)); err != nil { + t.Fatal(err) + } + p.Mode = api.ModeEnforce + if _, err := s.Replace(context.Background(), b, sectionPlan("r2", p)); err != nil { + t.Fatal(err) + } + if privilege.replacements != 2 { + t.Fatalf("mode change must replace the privilege instance, replacements=%d", privilege.replacements) + } +} + +func TestSectionBackendRetriesPrivilegeCleanup(t *testing.T) { + privilege := &fakeBackend{plans: map[string]Plan{}} + s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege) + b := Binding{ContainerID: "c"} + if _, err := s.Replace(context.Background(), b, sectionPlan("r1", CompiledPolicy{Privilege: privilegeSection(api.ActionAllow)})); err != nil { + t.Fatal(err) + } + privilege.withdrawPostCommit = true + r, err := s.Withdraw(context.Background(), b) + if err == nil || !r.Committed { + t.Fatalf("committed withdraw with cleanup error expected: %+v %v", r, err) + } + privilege.withdrawPostCommit = false + if r, err = s.Withdraw(context.Background(), b); err != nil || !r.Committed || privilege.withdrawals != 2 { + t.Fatalf("retry must reach the privilege backend: %+v %v withdrawals=%d", r, err, privilege.withdrawals) + } + if _, ok := s.active[bindingKey(b)]; ok { + t.Fatal("state must be gone after a clean retry") + } +} + +func TestSectionBackendPartialWithdrawIsUnconfirmed(t *testing.T) { + privilege := &fakeBackend{plans: map[string]Plan{}} + s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege) + b := Binding{ContainerID: "c"} + p := CompiledPolicy{Exec: api.ExecSection{Default: api.ActionDeny}, Privilege: privilegeSection(api.ActionAllow)} + if _, err := s.Replace(context.Background(), b, sectionPlan("r1", p)); err != nil { + t.Fatal(err) + } + privilege.fail = true + if r, _ := s.Withdraw(context.Background(), b); r.Committed || !r.ProtectionUnconfirmed { + t.Fatalf("exec removed but privilege not: %+v", r) + } +} + +func TestControllerPrepareScheduled(t *testing.T) { + ctx := context.Background() + withPrivilege := policy("with-privilege", "ns") + withPrivilege.Spec.Privilege = privilegeSection(api.ActionDeny) + execOnly := policy("exec-only", "ns") + c, _, _, _ := setup(t, withPrivilege, execOnly) + preparer := &preparingBackend{fakeBackend: fakeBackend{plans: map[string]Plan{}}} + c.backend = preparer + if err := c.InitialSync(ctx); err != nil { + t.Fatal(err) + } + template := binding() + template.ContainerID, template.CgroupID = "", 0 // a scheduled pod: no runtime identity yet + other := template + other.Namespace = "elsewhere" + if err := c.PrepareScheduled(ctx, []Binding{template, other}); err != nil { + t.Fatal(err) + } + if len(preparer.prepared) != 1 || len(preparer.prepared[0].Policies) != 1 || preparer.prepared[0].Policies[0].Identity.Name != "with-privilege" { + t.Fatalf("prepared %+v", preparer.prepared) + } +} diff --git a/pkg/sandboxpolicy/types.go b/pkg/sandboxpolicy/types.go index 752f184177..51c1865d17 100644 --- a/pkg/sandboxpolicy/types.go +++ b/pkg/sandboxpolicy/types.go @@ -45,6 +45,8 @@ type CompiledPolicy struct { Mode api.PolicyMode `json:"mode"` RootRole string `json:"rootRole"` Exec api.ExecSection `json:"exec"` + // Privilege is nil when the policy has no privilege section. + Privilege *api.PrivilegeSection `json:"privilege,omitempty"` } type Plan struct { @@ -101,6 +103,13 @@ type ProtectionAttester interface { Attest(context.Context, Binding, string) error } +// SlotPreparer optionally prepares enforcement for pods scheduled to this node +// before their containers start. Each plan is what a container of such a pod +// will get. It is a hint: a later Replace must still work without it. +type SlotPreparer interface { + Prepare(context.Context, []Plan) error +} + // SourceResolver reads ContainerProfile execs in the selected pod's namespace. // It must return an error for missing/incomplete profiles; empty successful results // mean a known complete profile with no execs. Rules retain their predicates. From 6c27bb0d6278b5201a7fadc4a697001ab4c87111 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:18:02 +0200 Subject: [PATCH 02/54] fix(enforce_privilege): address review comments - each namespace flag of a call resolves on its own (bpf_loop), so an allow for one flag cannot carry another flag past a default deny - compat ptrace read-mode access is not judged - failed arming marks are counted; a deep tree arms instead of matching a wrong namespace - transient arming failures are retried and reported - a binding whose rollback failed stays tracked for Unbind - reject NUL in mountFsType and duplicate slot generation entries - the gadget build loop stops on the first failure Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- Makefile | 2 +- pkg/ebpf/gadgets/common/arming.h | 29 ++++--- .../gadgets/common/enforcement/arm_linux.go | 20 ++++- .../gadgets/common/enforcement/bindings.go | 5 +- pkg/ebpf/gadgets/common/verdict.h | 2 +- pkg/ebpf/gadgets/enforce_privilege/README.md | 9 +- .../enforce_privilege/policy/kernel_test.go | 1 + .../enforce_privilege/policy/policy.go | 4 +- .../gadgets/enforce_privilege/program.bpf.c | 84 +++++++++++++++---- .../enforce_privilege/runtime/runtime.go | 9 +- .../enforce_privilege/runtime/runtime_test.go | 4 + 11 files changed, 126 insertions(+), 43 deletions(-) diff --git a/Makefile b/Makefile index 8fea1ceb69..8c4c5e8ba0 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ docker-push: docker-build docker push $(IMAGE):$(TAG) gadgets: - $(foreach img,$(KUBESCAPE_GADGETS),$(MAKE) -C ./pkg/ebpf/gadgets/$(img) build IMAGE=$(img) TAG=latest;) + $(foreach img,$(KUBESCAPE_GADGETS),$(MAKE) -C ./pkg/ebpf/gadgets/$(img) build IMAGE=$(img) TAG=latest || exit 1;) $(foreach img,$(BUILT_GADGETS),$(MAKE) -C ./pkg/ebpf/gadgets/$(img) build IMAGE=ghcr.io/inspektor-gadget/gadget/$(img) TAG=$(VERSION);) $(foreach img,$(GADGETS),sudo ig image pull ghcr.io/inspektor-gadget/gadget/$(img):$(VERSION);) sudo ig image export $(foreach img,$(GADGETS) $(BUILT_GADGETS),ghcr.io/inspektor-gadget/gadget/$(img):$(VERSION)) $(foreach img,$(KUBESCAPE_GADGETS),$(img):latest) tracers.tar diff --git a/pkg/ebpf/gadgets/common/arming.h b/pkg/ebpf/gadgets/common/arming.h index 4dda642b59..1ea8677d68 100644 --- a/pkg/ebpf/gadgets/common/arming.h +++ b/pkg/ebpf/gadgets/common/arming.h @@ -53,12 +53,16 @@ static __always_inline bool arming_task_armed(struct task_struct *task) return a && *a; } -static __always_inline void arming_mark(struct task_struct *task) +// arming_mark returns false when the task storage could not be allocated (memory +// pressure): the task then stays unarmed, and the caller counts the failure. +static __always_inline bool arming_mark(struct task_struct *task) { __u32 *a = bpf_task_storage_get(&armed_tasks, task, 0, BPF_LOCAL_STORAGE_GET_F_CREATE); - if (a) - *a = 1; + if (!a) + return false; + *a = 1; + return true; } #define ARMING_MAX_ANCESTORS 64 @@ -90,28 +94,31 @@ static __always_inline __u64 arming_host_mntns(struct task_struct *task) } t = parent; } - // Tree deeper than the walk: do not cache a value that may be wrong. - return BPF_CORE_READ(t, nsproxy, mnt_ns, ns.inum); + // Tree deeper than the walk: no namespace inode is 0, so the task is + // treated as outside the runtime namespace (armed), and nothing is cached. + return 0; } // arming_on_exec runs after a successful exec (bprm_committed_creds). -static __always_inline void arming_on_exec(void) +// arming_on_exec returns false when a needed mark failed. +static __always_inline bool arming_on_exec(void) { struct task_struct *task = bpf_get_current_task_btf(); __u64 mntns = BPF_CORE_READ(task, nsproxy, mnt_ns, ns.inum); if (arming_task_armed(task)) - return; + return true; if (mntns == arming_host_mntns(task)) - return; - arming_mark(task); + return true; + return arming_mark(task); } // arming_on_fork runs in task_alloc: the child inherits the parent's mark. -static __always_inline void arming_on_fork(struct task_struct *child) +static __always_inline bool arming_on_fork(struct task_struct *child) { if (arming_task_armed(bpf_get_current_task_btf())) - arming_mark(child); + return arming_mark(child); + return true; } #endif /* __ENFORCEMENT_ARMING_H */ diff --git a/pkg/ebpf/gadgets/common/enforcement/arm_linux.go b/pkg/ebpf/gadgets/common/enforcement/arm_linux.go index fea9596044..ef8b94e899 100644 --- a/pkg/ebpf/gadgets/common/enforcement/arm_linux.go +++ b/pkg/ebpf/gadgets/common/enforcement/arm_linux.go @@ -85,23 +85,35 @@ func armThreads(armed Map, threads string, hostMntns uint64, seen map[int]bool) if err != nil || seen[tid] { continue } - seen[tid] = true - if mntns, err := mntnsOf(scanner.Text()); err != nil || mntns == hostMntns { + // A task is seen once it is armed or needs no arming; any other + // failure is returned, so the caller does not report it protected. + mntns, err := mntnsOf(scanner.Text()) + if errors.Is(err, fs.ErrNotExist) || (err == nil && mntns == hostMntns) { + seen[tid] = true // exited, or a runtime helper continue } + if err != nil { + return fmt.Errorf("reading mount namespace of task %d: %w", tid, err) + } fd, err := unix.PidfdOpen(tid, pidfdThread) if errors.Is(err, unix.EINVAL) { fd, err = unix.PidfdOpen(tid, 0) } - if err != nil { - // The task exited, or it is a non-leader thread on an older kernel. + if errors.Is(err, unix.ESRCH) || errors.Is(err, unix.EINVAL) { + // Exited, or a non-leader thread on a kernel without + // PIDFD_THREAD: the BPF rule judges it through its leader. + seen[tid] = true continue } + if err != nil { + return fmt.Errorf("opening task %d: %w", tid, err) + } err = armed.Put(uint32(fd), uint32(1)) unix.Close(fd) if err != nil && !errors.Is(err, unix.ESRCH) { return fmt.Errorf("arming task %d: %w", tid, err) } + seen[tid] = true } return scanner.Err() } diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings.go b/pkg/ebpf/gadgets/common/enforcement/bindings.go index bccc68e614..d5bad43220 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings.go @@ -218,7 +218,10 @@ func (b *Bindings) bind(c Binding) error { // A container that is gone needs no protection; a startup // replay skips it (see start). if derr := b.members.Delete(id.CgroupID); derr != nil { - err = errors.Join(err, derr) + // The kernel entry stays: keep it tracked so Unbind can + // remove it later. + b.active[id.CgroupID] = c + return errors.Join(err, derr) } return err } diff --git a/pkg/ebpf/gadgets/common/verdict.h b/pkg/ebpf/gadgets/common/verdict.h index ea88abdedc..39d267af0b 100644 --- a/pkg/ebpf/gadgets/common/verdict.h +++ b/pkg/ebpf/gadgets/common/verdict.h @@ -38,7 +38,7 @@ enum verdict_counter { VERDICT_COUNT_SUBMITTED = 1, VERDICT_COUNT_DROPPED = 2, VERDICT_COUNT_ALLOWED = 3, - VERDICT_COUNT_SECTION_ERROR = 4, + VERDICT_COUNT_SECTION_ERROR = 4, // privilege: failed arming marks VERDICT_COUNT_RATE_LIMITED = 5, VERDICT_COUNT_MAX = 6, }; diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index e7eba05e86..30faeccc06 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -13,10 +13,11 @@ One instance runs per node: combination share it. Limits per node: 1024 containers, 64 slots, 512 policy map entries per slot, 16384 in total. - Binding or unbinding a container is a live write to the `bindings` map. -- A combination the running instance does not have (a new or edited policy) - replaces the node instance: the candidate starts with every slot in use, is - ready with every container bound, and then the old instance stops. Both - enforce during the overlap. Unused slots are dropped at the next replacement. +- The policy map is frozen per instance. A combination the running instance + does not have (a new or edited policy) needs a new instance: the owner starts + one with every slot in use and every container bound, and then stops the old + one, so both enforce during the overlap (the sandbox policy controller does + this). - An owner can create a slot before a container starts (for example when its pod is scheduled to the node), so the container binds with a live write. - Several policies on one container merge as the SandboxPolicy design says: the diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go index a7ebc481f8..55a97c12a3 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go @@ -131,6 +131,7 @@ func TestValidationErrors(t *testing.T) { "cap prefix": " - {name: r, target: {capability: [CAP_SYS_ADMIN]}, action: deny}\n", "bad action": " - {name: r, target: {syscall: [mount]}, action: block}\n", "long fstype": " - {name: r, target: {mountFsType: [abcdefghijklmnop]}, action: deny}\n", + "NUL fstype": " - {name: r, target: {mountFsType: [\"pro\\0c\"]}, action: deny}\n", "duplicate name": " - {name: r, target: {syscall: [mount]}, action: deny}\n - {name: r, target: {syscall: [bpf]}, action: deny}\n", "unknown field": " - {name: r, target: {path: [/x]}, action: deny}\n", } { diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go b/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go index f105c652ae..b44849ade6 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go @@ -124,8 +124,8 @@ func (t Target) validate() error { } } for _, fs := range t.MountFSType { - if fs == "" || len(fs) >= KernelNameLen { - return fmt.Errorf("filesystem type %q must be 1-%d bytes", fs, KernelNameLen-1) + if fs == "" || len(fs) >= KernelNameLen || strings.ContainsRune(fs, 0) { + return fmt.Errorf("filesystem type %q must be 1-%d bytes without NUL", fs, KernelNameLen-1) } } return nil diff --git a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c index e731de0ab0..f1b04a7b6a 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c +++ b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c @@ -289,25 +289,72 @@ static __always_inline void consider_syscall(__u32 slot, enum priv_syscall sc, priv_consider(&key, m); } -// resolve_syscall resolves the syscall kind: rules on the syscall with any -// flag, rules on each CLONE_NEW* flag in flags, then the syscall default. +struct resolve_flags_ctx { + __u32 slot; + enum priv_syscall sc; + __u64 flags; + struct policy_match base; + struct policy_match out; +}; + +static __always_inline __u64 ns_flag_at(__u32 i) +{ + switch (i) { + case 0: + return CLONE_NEWNS; + case 1: + return CLONE_NEWCGROUP; + case 2: + return CLONE_NEWUTS; + case 3: + return CLONE_NEWIPC; + case 4: + return CLONE_NEWUSER; + case 5: + return CLONE_NEWPID; + case 6: + return CLONE_NEWNET; + case 7: + return CLONE_NEWTIME; + } + return 0; +} + +// resolve_flag is a bpf_loop callback: one namespace flag of the call. +static long resolve_flag(__u32 i, void *data) +{ + struct resolve_flags_ctx *c = data; + __u64 flag = ns_flag_at(i); + struct policy_match m; + + if (!flag) + return 1; + if (!(c->flags & flag)) + return 0; + m = c->base; + consider_syscall(c->slot, c->sc, flag, &m); + priv_finish_kind(c->slot, PRIV_KIND_SYSCALL, &m); + policy_merge(&c->out, &m); + return 0; +} + +// resolve_syscall resolves the syscall kind. Each namespace flag of a call +// resolves on its own (rules on the flag or on the syscall with any flag, else +// the default), so an allow for one flag never lets another flag of the same +// call pass. The per-flag work runs in bpf_loop to keep verification small. static __always_inline void resolve_syscall(__u32 slot, enum priv_syscall sc, __u64 flags, struct policy_match *out) { - static const __u64 ns_flags[] = { - CLONE_NEWNS, CLONE_NEWCGROUP, CLONE_NEWUTS, CLONE_NEWIPC, - CLONE_NEWUSER, CLONE_NEWPID, CLONE_NEWNET, CLONE_NEWTIME, - }; - struct policy_match m = {}; + struct resolve_flags_ctx c = { .slot = slot, .sc = sc, .flags = flags }; - consider_syscall(slot, sc, 0, &m); -#pragma unroll - for (int i = 0; i < sizeof(ns_flags) / sizeof(ns_flags[0]); i++) { - if (flags & ns_flags[i]) - consider_syscall(slot, sc, ns_flags[i], &m); + consider_syscall(slot, sc, 0, &c.base); + if (!(flags & CLONE_NS_MASK)) { + priv_finish_kind(slot, PRIV_KIND_SYSCALL, &c.base); + policy_merge(out, &c.base); + return; } - priv_finish_kind(slot, PRIV_KIND_SYSCALL, &m); - policy_merge(out, &m); + bpf_loop(8, resolve_flag, &c, 0); + policy_merge(out, &c.out); } static __always_inline void resolve_capability(__u32 slot, int cap, @@ -416,7 +463,8 @@ static __always_inline int check_syscall(void *ctx, enum priv_hook hook, SEC("lsm/bprm_committed_creds") int BPF_PROG(ig_priv_exec) { - arming_on_exec(); + if (!arming_on_exec()) + verdict_count(VERDICT_COUNT_SECTION_ERROR); return 0; } @@ -462,7 +510,8 @@ static __always_inline int do_task_alloc(void *ctx, struct task_struct *task, enum priv_syscall sc; __u64 flags; - arming_on_fork(task); + if (!arming_on_fork(task)) + verdict_count(VERDICT_COUNT_SECTION_ERROR); // userns_create judges CLONE_NEWUSER. flags = clone_flags & CLONE_NS_MASK & ~CLONE_NEWUSER; @@ -696,6 +745,9 @@ static __always_inline int do_ptrace_access_check(void *ctx, unsigned int mode) __u32 slot = 0; if (current_is_compat()) { + // Read-mode /proc access (ps, top) is never ptrace. + if (!(mode & PTRACE_MODE_ATTACH)) + return 0; // Unknown syscall: apply all three ptrace-family syscalls. header = priv_scope(&cgid, &slot); if (!header) diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go index 8561476cf3..df8501e046 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go @@ -57,7 +57,7 @@ func validateConfig(config Config) error { if p.Generation == 0 || p.Slots > privpolicy.KernelMaxSlots || len(p.Entries) > privpolicy.KernelMaxEntries { return fmt.Errorf("invalid compiled privilege policy") } - metas := uint32(0) + metas := map[uint32]bool{} for _, e := range p.Entries { if e.Key.Slot >= p.Slots { return fmt.Errorf("policy entry for slot %d out of range", e.Key.Slot) @@ -66,10 +66,13 @@ func validateConfig(config Config) error { if e.Value.Generation != p.Generation { return fmt.Errorf("slot %d carries another generation", e.Key.Slot) } - metas++ + if metas[e.Key.Slot] { + return fmt.Errorf("slot %d has two generation entries", e.Key.Slot) + } + metas[e.Key.Slot] = true } } - if metas != p.Slots { + if uint32(len(metas)) != p.Slots { return fmt.Errorf("every slot needs one generation entry") } return nil diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go index ae02b6a24e..97a251a2c7 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_test.go @@ -62,4 +62,8 @@ func TestValidateConfig(t *testing.T) { if err := validateConfig(Config{Policy: c}); err == nil { t.Fatal("a slot without its generation entry must fail") } + c.Entries = append(c.Entries, c.Entries[0]) // a second slot-0 entry, no slot 1 + if err := validateConfig(Config{Policy: c}); err == nil { + t.Fatal("duplicate generation entries must fail") + } } From c6b7570b77ee4ca907bcd4a9c94d1f6f9137b940 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:18:13 +0200 Subject: [PATCH 03/54] feat(sandboxpolicy): log whether a privilege bind found its slot ready Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/igbackend/privilege.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/sandboxpolicy/igbackend/privilege.go b/pkg/sandboxpolicy/igbackend/privilege.go index cbcd790d86..0b1f620945 100644 --- a/pkg/sandboxpolicy/igbackend/privilege.go +++ b/pkg/sandboxpolicy/igbackend/privilege.go @@ -10,6 +10,8 @@ import ( "sort" "sync" + "github.com/kubescape/go-logger" + "github.com/kubescape/go-logger/helpers" "github.com/kubescape/node-agent/pkg/ebpf/gadgets/common/enforcement" privpolicy "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/policy" privruntime "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/runtime" @@ -124,6 +126,13 @@ func Combination(plan sandboxpolicy.Plan) (string, []sandboxpolicy.CompiledPolic return hex.EncodeToString(sum[:]), policies } +// logBind records whether a new container found its slot ready ("prepared") +// or needed a node instance replacement ("replaced"). +func logBind(b sandboxpolicy.Binding, slot string) { + logger.L().Info("sandbox privilege bind", helpers.String("slot", slot), + helpers.String("namespace", b.Namespace), helpers.String("pod", b.PodName), helpers.String("container", b.ContainerName)) +} + func (b *PrivilegeBackend) Stats() PrivilegeStats { b.mu.Lock() defer b.mu.Unlock() @@ -186,6 +195,7 @@ func (b *PrivilegeBackend) Replace(ctx context.Context, binding sandboxpolicy.Bi } if current == nil { b.stats.Prepared++ + logBind(binding, "prepared") } b.bindings[k] = next return commit() @@ -201,6 +211,7 @@ func (b *PrivilegeBackend) Replace(ctx context.Context, binding sandboxpolicy.Bi } if current == nil { b.stats.Replaced++ + logBind(binding, "replaced") } return commit() } From 1092ae8ee21b5afc2429d7a703489c7081ae5b17 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:20:59 +0200 Subject: [PATCH 04/54] docs(enforce_privilege): document the known limits Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/enforce_privilege/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index 30faeccc06..8cbaa976c8 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -94,6 +94,17 @@ spec: candidate syscall rule applies. - **Deny** returns `-EPERM` from the LSM hook, before the side effect. +## Known limits + +- **Failed arming mark.** If the kernel cannot allocate task storage (memory + pressure), the task stays unarmed and is not judged until its next exec. The + failure is counted in `counters` slot 4. A later check cannot tell such a + task apart from the runtime's own setup, so it is not retried. +- **No-audit capability checks** (`CAP_OPT_NOAUDIT`) are not judged. They are + kernel probes (for example the overcommit check); judging them would deny + normal work in every bound container. One of them grants write access to + `/proc/sys/kernel/ns_last_pid`, which the privilege section does not cover. + ## Maps and events | Map | Content | From 141b7e041e9d1d93053a8efa7defa3d9ef20c5ac Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:22:41 +0200 Subject: [PATCH 05/54] docs(enforce_privilege): correct the no-audit limit Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/enforce_privilege/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index 8cbaa976c8..b474cdf23e 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -100,10 +100,11 @@ spec: pressure), the task stays unarmed and is not judged until its next exec. The failure is counted in `counters` slot 4. A later check cannot tell such a task apart from the runtime's own setup, so it is not retried. -- **No-audit capability checks** (`CAP_OPT_NOAUDIT`) are not judged. They are - kernel probes (for example the overcommit check); judging them would deny - normal work in every bound container. One of them grants write access to - `/proc/sys/kernel/ns_last_pid`, which the privilege section does not cover. +- **No-audit capability checks** (`CAP_OPT_NOAUDIT`) are excluded from + capability enforcement. Most are kernel probes (for example the overcommit + check), and judging them would deny normal work in every bound container. Some + also grant access (in current kernels, writing the pid namespace's `pid_max`), + so `capability` rules do not cover every `CAP_SYS_ADMIN` use. ## Maps and events From 02c40ef1a5e1acad6574a0d5b05d4f33f558db19 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:11:27 +0200 Subject: [PATCH 06/54] fix(sandboxpolicy): read the process cgroup in the host cgroup namespace The binding cgroup path comes from the container collection, which reads /proc//cgroup in the cgroup namespace of PID 1. node-agent runs in its own cgroup namespace, where the same file shows a /../.. path, so every lease check failed with "process cgroup changed" and no container bound. Check now reads the file in PID 1's cgroup namespace when it differs. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/igbackend/identity_linux.go | 67 +++++++++++++++++-- .../igbackend/identity_linux_test.go | 27 ++++++++ 2 files changed, 89 insertions(+), 5 deletions(-) diff --git a/pkg/sandboxpolicy/igbackend/identity_linux.go b/pkg/sandboxpolicy/igbackend/identity_linux.go index b6740883a5..36957fe230 100644 --- a/pkg/sandboxpolicy/igbackend/identity_linux.go +++ b/pkg/sandboxpolicy/igbackend/identity_linux.go @@ -8,6 +8,7 @@ import ( "os" "path" "path/filepath" + "runtime" "strconv" "strings" @@ -21,8 +22,13 @@ import ( type LinuxValidator struct{ ProcRoot, CgroupRoot string } type linuxLease struct { proc, cgroup, namespace *os.File - identity sandboxpolicy.Binding - start string + // hostCgroupNS is the cgroup namespace of PID 1 of procRoot when it is not + // ours. /proc//cgroup is relative to the reader's cgroup namespace, + // and the binding's path is relative to PID 1's (as the container + // collection reads it), so Check reads it there. + hostCgroupNS *os.File + identity sandboxpolicy.Binding + start string } func (v LinuxValidator) Acquire(ctx context.Context, b sandboxpolicy.Binding) (Lease, error) { @@ -86,6 +92,9 @@ func (v LinuxValidator) Acquire(ctx context.Context, b sandboxpolicy.Binding) (L if stat.Ino != b.MountNamespace { return fail(fmt.Errorf("mount namespace changed")) } + if lease.hostCgroupNS, err = foreignCgroupNamespace(procRoot); err != nil { + return fail(err) + } lease.start, err = lease.readStart() if err != nil { return fail(err) @@ -136,7 +145,7 @@ func (l *linuxLease) Check() error { if start != l.start { return fmt.Errorf("process identity changed") } - data, err := l.read("cgroup") + data, err := l.readCgroup() if err != nil { return err } @@ -168,11 +177,59 @@ func (l *linuxLease) Check() error { } func (l *linuxLease) Close() error { var errs []error - for _, f := range []*os.File{l.namespace, l.cgroup, l.proc} { + for _, f := range []*os.File{l.hostCgroupNS, l.namespace, l.cgroup, l.proc} { if f != nil { errs = append(errs, f.Close()) } } - l.namespace, l.cgroup, l.proc = nil, nil, nil + l.hostCgroupNS, l.namespace, l.cgroup, l.proc = nil, nil, nil, nil return errors.Join(errs...) } + +// foreignCgroupNamespace opens the cgroup namespace of PID 1 of procRoot. It +// returns nil when this process is already in it. node-agent normally runs in +// its own (private) cgroup namespace, where /proc//cgroup of a workload +// shows a "/../.." path instead of the path the container collection recorded. +func foreignCgroupNamespace(procRoot string) (*os.File, error) { + host, err := os.Open(filepath.Join(procRoot, "1", "ns", "cgroup")) + if err != nil { + return nil, fmt.Errorf("host cgroup namespace: %w", err) + } + var hostStat, selfStat unix.Stat_t + if err := unix.Fstat(int(host.Fd()), &hostStat); err != nil { + return nil, errors.Join(err, host.Close()) + } + if err := unix.Stat("/proc/thread-self/ns/cgroup", &selfStat); err != nil { + return nil, errors.Join(err, host.Close()) + } + if hostStat.Dev == selfStat.Dev && hostStat.Ino == selfStat.Ino { + return nil, host.Close() + } + return host, nil +} + +// readCgroup reads /proc//cgroup in the host cgroup namespace. +func (l *linuxLease) readCgroup() ([]byte, error) { + if l.hostCgroupNS == nil { + return l.read("cgroup") + } + runtime.LockOSThread() + own, err := os.Open("/proc/thread-self/ns/cgroup") + if err != nil { + runtime.UnlockOSThread() + return nil, err + } + defer own.Close() + if err := unix.Setns(int(l.hostCgroupNS.Fd()), unix.CLONE_NEWCGROUP); err != nil { + runtime.UnlockOSThread() + return nil, fmt.Errorf("enter host cgroup namespace: %w", err) + } + data, readErr := l.read("cgroup") + if err := unix.Setns(int(own.Fd()), unix.CLONE_NEWCGROUP); err != nil { + // Keep the thread locked: Go ends it when this goroutine exits, so no + // other goroutine runs in the wrong cgroup namespace. + return nil, fmt.Errorf("restore cgroup namespace: %w", err) + } + runtime.UnlockOSThread() + return data, readErr +} diff --git a/pkg/sandboxpolicy/igbackend/identity_linux_test.go b/pkg/sandboxpolicy/igbackend/identity_linux_test.go index b3d5d0eebd..4fa9cd65cc 100644 --- a/pkg/sandboxpolicy/igbackend/identity_linux_test.go +++ b/pkg/sandboxpolicy/igbackend/identity_linux_test.go @@ -139,3 +139,30 @@ func TestLinuxLeaseRejectsCgroupMovementBeforeNamespaceAccess(t *testing.T) { }) } } + +func TestForeignCgroupNamespace(t *testing.T) { + link := func(target string) string { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "1", "ns"), 0700); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, filepath.Join(root, "1", "ns", "cgroup")); err != nil { + t.Fatal(err) + } + return root + } + // In our own cgroup namespace there is nothing to enter. + if ns, err := foreignCgroupNamespace(link("/proc/thread-self/ns/cgroup")); err != nil || ns != nil { + t.Fatalf("own namespace: ns=%v err=%v", ns, err) + } + // A different namespace is kept open for Check. + ns, err := foreignCgroupNamespace(link("/proc/thread-self/ns/mnt")) + if err != nil || ns == nil { + t.Fatalf("foreign namespace: ns=%v err=%v", ns, err) + } + ns.Close() + // A missing host namespace fails closed. + if _, err := foreignCgroupNamespace(filepath.Join(t.TempDir(), "missing")); err == nil { + t.Fatal("missing host cgroup namespace accepted") + } +} From e716b2df4e74abc5e7f46e0c0408820a5273e204 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:17:05 +0200 Subject: [PATCH 07/54] fix(sandboxpolicy): pass host-mntns-id under the oci ebpf operator prefix Gadget eBPF parameters are named operator.oci.ebpf.. The engine set operator.ebpf.host-mntns-id, which IG ignored, so host_mntns_id stayed 0 and the program used the global init task's mount namespace. In kind the runtime runs in the node container's mount namespace, so runc init for kubectl exec was armed and its setns(CLONE_NEWNS) was denied: every exec into a bound pod failed. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/igbackend/privilege_native.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sandboxpolicy/igbackend/privilege_native.go b/pkg/sandboxpolicy/igbackend/privilege_native.go index 86a20b8ac5..141df05083 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_native.go +++ b/pkg/sandboxpolicy/igbackend/privilege_native.go @@ -128,7 +128,7 @@ func (e *nativePrivilegeEngine) Start(ctx context.Context, id string, config pri return nil, fmt.Errorf("runtime mount namespace: %w", err) } desired := &api.GadgetInstance{Id: id, GadgetConfig: &api.GadgetRunRequest{Version: api.VersionGadgetRunProtocol, ImageName: e.image, - ParamValues: map[string]string{"operator.ebpf.host-mntns-id": strconv.FormatUint(runtimeMntns, 10)}}} + ParamValues: map[string]string{"operator.oci.ebpf.host-mntns-id": strconv.FormatUint(runtimeMntns, 10)}}} err = e.manager.StartGadget(ctx, desired, gadgetcontext.WithOrasReadonlyTarget(e.store), gadgetcontext.WithAdditionalDataOperators(handler, e.kube, privruntime.New(), events), From 2e8a266cdcc985c90d1f394bd0c55515b6df1ac3 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 01:18:02 +0200 Subject: [PATCH 08/54] fix(enforcement): report cgroup listing errors other than a removed cgroup Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/common/enforcement/arm_linux.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/ebpf/gadgets/common/enforcement/arm_linux.go b/pkg/ebpf/gadgets/common/enforcement/arm_linux.go index ef8b94e899..699fb6d4ed 100644 --- a/pkg/ebpf/gadgets/common/enforcement/arm_linux.go +++ b/pkg/ebpf/gadgets/common/enforcement/arm_linux.go @@ -51,7 +51,10 @@ func ArmExisting(armed Map, cgroupPath string) error { } return fmt.Errorf("listing container tasks: %w", err) } - return nil // a child cgroup removed during the walk + if errors.Is(err, fs.ErrNotExist) { + return nil // a child cgroup removed during the walk + } + return fmt.Errorf("listing container tasks: %w", err) } if !d.IsDir() { return nil @@ -75,9 +78,12 @@ const maxArmingPasses = 8 func armThreads(armed Map, threads string, hostMntns uint64, seen map[int]bool) error { f, err := os.Open(threads) - if err != nil { + if errors.Is(err, fs.ErrNotExist) { return nil // the cgroup was removed } + if err != nil { + return fmt.Errorf("listing container tasks: %w", err) + } defer f.Close() scanner := bufio.NewScanner(f) for scanner.Scan() { From 06c4336d93c523db0d5d8c1eaac5867b0decae05 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 01:18:02 +0200 Subject: [PATCH 09/54] fix(sandboxpolicy): address review comments on the controller connection - keep an unready candidate that failed to stop for a stop retry - prepare slots for ephemeral containers too - list the node's pods for slot preparation only after a Pending-pod event or every 30 s, not on every refresh Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/igbackend/privilege.go | 6 ++++- pkg/sandboxpolicy/igbackend/privilege_test.go | 23 +++++++++++++++++++ pkg/sandboxpolicy/integration/bindings.go | 5 +++- pkg/sandboxpolicy/integration/service.go | 14 ++++++++++- 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/pkg/sandboxpolicy/igbackend/privilege.go b/pkg/sandboxpolicy/igbackend/privilege.go index 0b1f620945..d85e24972b 100644 --- a/pkg/sandboxpolicy/igbackend/privilege.go +++ b/pkg/sandboxpolicy/igbackend/privilege.go @@ -330,7 +330,11 @@ func (b *PrivilegeBackend) replaceInstance(ctx context.Context, prepared, extra return err } if err = candidate.Ready(); err != nil { - return errors.Join(err, candidate.Stop(ctx)) + if stopErr := candidate.Stop(ctx); stopErr != nil { + b.retiring = append(b.retiring, candidate) // retried by retire + return errors.Join(err, stopErr) + } + return err } if b.instance != nil { b.retiring = append(b.retiring, b.instance) diff --git a/pkg/sandboxpolicy/igbackend/privilege_test.go b/pkg/sandboxpolicy/igbackend/privilege_test.go index 171a976fea..1a83423292 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_test.go +++ b/pkg/sandboxpolicy/igbackend/privilege_test.go @@ -214,3 +214,26 @@ func TestPrivilegeOversizedPrepareDoesNotBlockContainers(t *testing.T) { t.Fatalf("a real container must still bind: %+v %v", r, err) } } + +func TestPrivilegeUnreadyCandidateThatFailsToStopIsRetried(t *testing.T) { + e := &readyFailEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + if r, err := b.Replace(ctx, privBinding(1), privPlan("r1", privPolicy("p1", api.ActionAllow))); err == nil || r.Committed { + t.Fatalf("an unready candidate must fail: %+v", r) + } + if len(b.retiring) != 1 { + t.Fatal("a candidate that failed to stop must be kept for retry") + } + e.instance.stopErr = nil + if err := b.Close(ctx); err != nil || e.instance.stops != 2 { + t.Fatalf("close must retry the stop: %v stops=%d", err, e.instance.stops) + } +} + +type readyFailEngine struct{ instance *fakePrivilegeInstance } + +func (e *readyFailEngine) Start(context.Context, string, privruntime.Config) (PrivilegeInstance, error) { + e.instance = &fakePrivilegeInstance{readyErr: errors.New("not ready"), stopErr: errors.New("busy"), bound: map[uint64]uint32{}} + return e.instance, nil +} diff --git a/pkg/sandboxpolicy/integration/bindings.go b/pkg/sandboxpolicy/integration/bindings.go index 59808445f5..5e5e332515 100644 --- a/pkg/sandboxpolicy/integration/bindings.go +++ b/pkg/sandboxpolicy/integration/bindings.go @@ -62,7 +62,7 @@ func SnapshotSelectedBindingsFromContainers(ctx context.Context, client kubernet } // ScheduledContainers returns API metadata (no runtime identity) for every -// declared regular and init container of the pods scheduled to this node that +// declared regular, init and ephemeral container of the pods scheduled to this node that // are not finished. It feeds privilege slot preparation, not binding. func ScheduledContainers(ctx context.Context, client kubernetes.Interface, nodeName string) ([]sandboxpolicy.Binding, error) { if client == nil || nodeName == "" { @@ -97,6 +97,9 @@ func ScheduledContainers(ctx context.Context, client kubernetes.Interface, nodeN result = append(result, sandboxpolicy.Binding{PodUID: pod.UID, Namespace: pod.Namespace, PodName: pod.Name, ContainerName: c.Name, Kind: group.kind, PodLabels: copyLabels(pod.Labels), NamespaceLabels: copyLabels(labels)}) } } + for _, c := range pod.Spec.EphemeralContainers { + result = append(result, sandboxpolicy.Binding{PodUID: pod.UID, Namespace: pod.Namespace, PodName: pod.Name, ContainerName: c.Name, Kind: "ephemeral", PodLabels: copyLabels(pod.Labels), NamespaceLabels: copyLabels(labels)}) + } } return result, nil } diff --git a/pkg/sandboxpolicy/integration/service.go b/pkg/sandboxpolicy/integration/service.go index 507ef596a1..51344e171e 100644 --- a/pkg/sandboxpolicy/integration/service.go +++ b/pkg/sandboxpolicy/integration/service.go @@ -73,8 +73,15 @@ type Service struct { wake chan struct{} inputEpoch atomic.Uint64 snapshotEpoch atomic.Uint64 + // podsScheduled is set by a Pending-pod event; lastPrepare bounds how + // often privilege slot preparation lists the node's pods. + podsScheduled atomic.Bool + lastPrepare time.Time } +// prepareInterval is the longest time between two slot preparations. +const prepareInterval = 30 * time.Second + // NewService prepares the public watcher/exporter/native runtime bridge. It does // not authorize any cgroup or start enforcement until Start is called. func NewService(ctx context.Context, options ServiceOptions) (*Service, error) { @@ -176,6 +183,7 @@ func (s *Service) configurationHint(obj runtime.Object) { } if pod, ok := obj.(*corev1.Pod); ok { if s.options.PrivilegeImageName != "" && pod.Spec.NodeName == s.options.NodeName && pod.Status.Phase == corev1.PodPending { + s.podsScheduled.Store(true) s.ContainerCallback(containercollection.PubSubEvent{}) } return @@ -243,7 +251,11 @@ drain: } func (s *Service) refresh(ctx context.Context) { s.snapshotEpoch.Store(s.inputEpoch.Load()) - if s.options.PrivilegeImageName != "" && s.options.Kubernetes != nil && s.Controller.HasPrivilegePolicies() { + // Prepare after a Pending-pod event, else at most every prepareInterval, + // so the extra pod and namespace listing does not run on every refresh. + if s.options.PrivilegeImageName != "" && s.options.Kubernetes != nil && s.Controller.HasPrivilegePolicies() && + (s.podsScheduled.Swap(false) || time.Since(s.lastPrepare) >= prepareInterval) { + s.lastPrepare = time.Now() scheduled, err := ScheduledContainers(ctx, s.options.Kubernetes, s.options.NodeName) if err == nil { err = s.Controller.PrepareScheduled(ctx, scheduled) From 6cb5100ffaafb5b909d5104e27c0d0a8b80cca8a Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 10:03:22 +0200 Subject: [PATCH 10/54] refactor(enforce_privilege): make the gadget and common library portable - the gadget has its own policy document with Parse and an example file; the SandboxPolicy translation moves to the controller connection (#1068) - neutral names in the gadget, BPF, gadget.yaml and README - a guard test fails on imports outside Inspektor Gadget and the gadget folders, and on host-product names or outside includes - an opt-in test runs the gadget with Inspektor Gadget alone Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/common/arming.h | 2 +- .../common/enforcement/portability_test.go | 116 +++++++++++++++ pkg/ebpf/gadgets/enforce_privilege/README.md | 75 ++++++---- .../enforce_privilege/examples/policy.yaml | 28 ++++ .../gadgets/enforce_privilege/gadget.yaml | 2 +- .../gadgets/enforce_privilege/policy/api.go | 24 --- .../enforce_privilege/policy/kernel.go | 8 +- .../enforce_privilege/policy/kernel_test.go | 38 +++-- .../enforce_privilege/policy/policy.go | 36 ++++- .../gadgets/enforce_privilege/program.bpf.c | 3 +- .../runtime/runtime_kernel_test.go | 139 ++++++++++++++++++ 11 files changed, 396 insertions(+), 75 deletions(-) create mode 100644 pkg/ebpf/gadgets/common/enforcement/portability_test.go create mode 100644 pkg/ebpf/gadgets/enforce_privilege/examples/policy.yaml delete mode 100644 pkg/ebpf/gadgets/enforce_privilege/policy/api.go create mode 100644 pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go diff --git a/pkg/ebpf/gadgets/common/arming.h b/pkg/ebpf/gadgets/common/arming.h index 1ea8677d68..d845e3d08a 100644 --- a/pkg/ebpf/gadgets/common/arming.h +++ b/pkg/ebpf/gadgets/common/arming.h @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright 2026 The Kubescape Authors -// Arming rule shared by the SandboxPolicy prevention gadgets. +// Arming rule shared by the BPF LSM enforcement gadgets. // // A cgroup-scoped enforcer also sees the container runtime's own setup inside // the container cgroup (runc init creates namespaces, mounts, pivots root and, diff --git a/pkg/ebpf/gadgets/common/enforcement/portability_test.go b/pkg/ebpf/gadgets/common/enforcement/portability_test.go new file mode 100644 index 0000000000..7b5dee9477 --- /dev/null +++ b/pkg/ebpf/gadgets/common/enforcement/portability_test.go @@ -0,0 +1,116 @@ +// Copyright 2026 The Kubescape Authors +// SPDX-License-Identifier: Apache-2.0 + +package enforcement + +import ( + "go/parser" + "go/token" + "io/fs" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" +) + +// The gadget folders must move into Inspektor Gadget with only an import-path +// rewrite. So they may import the standard library, Inspektor Gadget (and its +// enforce_exec types), the few libraries Inspektor Gadget itself uses, and each +// other — nothing else from the host repository and no Kubernetes or policy-CRD +// concepts. +var gadgetFolders = []string{"..", "../../enforce_privilege"} + +const ownModule = "github.com/kubescape/node-agent/pkg/ebpf/gadgets/" + +var allowedImports = []string{ + "github.com/inspektor-gadget/inspektor-gadget/", + "github.com/cilium/ebpf", + "github.com/spf13/viper", + "golang.org/x/sys/", + "gopkg.in/yaml.v3", + "oras.land/oras-go/v2", // Inspektor Gadget's OCI store, used by a test + ownModule + "common/", + ownModule + "enforce_privilege/", +} + +// Names of the host product and its policy API, which the gadgets must not use. +var forbiddenWords = regexp.MustCompile(`(?i)sandbox[ _-]?polic|kubescape\.io|\barmo(sec)?\b|node-agent|kubescape/storage|\bkubernetes\b|\bpods?\b`) + +// A BPF include must stay inside the gadget folders: "x.h" or "../common/x.h". +var badInclude = regexp.MustCompile(`#include\s+"(\.\./){2,}|#include\s+"\.\./[^c]`) + +func walkGadgetFiles(t *testing.T, fn func(path string, data []byte)) { + t.Helper() + for _, root := range gadgetFolders { + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if d.Name() == "lab" { // lab harness, not part of the gadget + return filepath.SkipDir + } + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + fn(path, data) + return nil + }) + if err != nil { + t.Fatal(err) + } + } +} + +func TestGadgetImportsArePortable(t *testing.T) { + walkGadgetFiles(t, func(path string, data []byte) { + if !strings.HasSuffix(path, ".go") { + return + } + f, err := parser.ParseFile(token.NewFileSet(), path, data, parser.ImportsOnly) + if err != nil { + t.Fatal(err) + } + for _, imp := range f.Imports { + p, _ := strconv.Unquote(imp.Path.Value) + if !strings.Contains(strings.SplitN(p, "/", 2)[0], ".") { + continue // standard library + } + ok := false + for _, allowed := range allowedImports { + if p == strings.TrimSuffix(allowed, "/") || strings.HasPrefix(p, allowed) { + ok = true + } + } + if !ok { + t.Errorf("%s imports %s, which a portable gadget must not use", path, p) + } + } + }) +} + +func TestGadgetFilesUseNeutralNames(t *testing.T) { + walkGadgetFiles(t, func(path string, data []byte) { + if filepath.Base(path) == "portability_test.go" && filepath.Base(filepath.Dir(path)) == "enforcement" { + return // this guard lists the forbidden names itself + } + for i, line := range strings.Split(string(data), "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, `"`+ownModule) || strings.Contains(trimmed, ` "`+ownModule) || + strings.HasPrefix(trimmed, "// Copyright") || strings.HasPrefix(trimmed, "# Copyright") { + continue // import paths are rewritten on a move; license headers stay + } + if badInclude.MatchString(line) { + t.Errorf("%s:%d includes a header outside the gadget folders", path, i+1) + } + if m := forbiddenWords.FindString(line); m != "" { + t.Errorf("%s:%d uses %q: gadget files must use neutral names", path, i+1, m) + } + } + }) +} diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index b474cdf23e..bb1e844fa0 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -1,35 +1,27 @@ # enforce_privilege -Container privilege enforcement with BPF LSM, no seccomp. It enforces the -`privilege` section of a sandbox policy: namespace creation and joining, mounts, -`pivot_root`/`chroot`, capabilities, ptrace, kernel modules, `bpf`, -`perf_event_open`, keys, `io_uring_setup`, `userfaultfd`, `mknod`, -`setuid`/`setgid` and `capset`. +Container privilege enforcement with BPF LSM, no seccomp. It enforces a +privilege policy: namespace creation and joining, mounts, `pivot_root`/`chroot`, +capabilities, ptrace, kernel modules, `bpf`, `perf_event_open`, keys, +`io_uring_setup`, `userfaultfd`, `mknod`, `setuid`/`setgid` and `capset`. The gadget is built on the [common enforcement library](../common/README.md). -One instance runs per node: +It depends only on Inspektor Gadget and that library. One instance runs per +node: -- A **slot** is one combination of privilege policies; containers with the same +- A **slot** is one combination of policy documents; containers with the same combination share it. Limits per node: 1024 containers, 64 slots, 512 policy map entries per slot, 16384 in total. - Binding or unbinding a container is a live write to the `bindings` map. - The policy map is frozen per instance. A combination the running instance does not have (a new or edited policy) needs a new instance: the owner starts one with every slot in use and every container bound, and then stops the old - one, so both enforce during the overlap (the sandbox policy controller does - this). -- An owner can create a slot before a container starts (for example when its - pod is scheduled to the node), so the container binds with a live write. -- Several policies on one container merge as the SandboxPolicy design says: the - union of rules (explicit deny > allow > default, each rule with its own name, - origin and mode); a default is deny if any policy denies, with the strictest - mode among the policies that agree. One addition to the design: a monitoring - (audit) deny rule never weakens another policy's enforce deny, including its - enforce default deny. An explicit allow still beats another policy's default - deny. - -`policy/` compiles the `privilege` section of one or more policies; -`runtime/` is the native owner a controller drives. + one, so both enforce during the overlap. +- An owner can create a slot before a container starts, so the container binds + with a live write. + +`policy/` parses and compiles policy documents; `runtime/` is the native +operator an owner drives with `runtime.WithConfig` and `runtime.GetBindings`. Requires cgroup v2, kernel BTF and BPF LSM enabled (`bpf` in `/sys/kernel/security/lsm`). Linux 6.1 or later (the `userns_create` hook; @@ -39,38 +31,46 @@ README). ## Policy -The `privilege` section of a SandboxPolicy, for example: +A policy document (YAML or JSON), parsed with `policy.Parse`. Full example: +[examples/policy.yaml](examples/policy.yaml). ```yaml -spec: +version: 1 +name: example # qualifies rule names in events +policy: mode: enforce # audit | enforce | off privilege: - default: allow # applies to syscall and mountFsType targets + default: allow # default for syscall and mountFsType targets rules: - name: no-new-namespaces target: {syscall: [unshare, clone, clone3, setns], flags: [CLONE_NEWUSER, CLONE_NEWNS]} action: deny - origin: catalog/G1 - name: no-proc-mount target: {mountFsType: [proc, sysfs]} action: deny - - name: audit-sys-admin + - name: watch-sys-admin target: {capability: [SYS_ADMIN]} action: deny - mode: audit + mode: audit # a rule may override the document mode ``` - A rule targets exactly one kind: `syscall` (optionally with `flags` for `unshare`, `setns`, `clone`, `clone3`), `capability` (short names such as `SYS_ADMIN`) or `mountFsType`. - Resolution: an explicit deny beats an allow, an allow beats the default, - independent of rule order. Each rule has its own mode. + independent of rule order. Each rule has its own mode. Each namespace flag of + a call resolves on its own. - Capability checks have no default; only explicit capability rules apply. - A `default: deny` also denies `setuid`/`setgid` (package managers drop privileges), implicit kernel module loads (judged as `init_module`, for example an `iptables` match module) and new superblocks of any filesystem type. Allow what the workload needs. -- `actor`, `except` and `pathClass` are not supported yet. +- **Several documents in one slot** combine so that the most limiting result + wins: the union of rules (explicit deny > allow > default, each rule with its + own name, origin and mode); a default is deny if any document denies, with the + strictest mode among the documents that agree; a monitoring (audit) deny rule + never weakens another document's enforce deny, including its enforce default + deny; an explicit allow still beats another document's default deny. ## How it decides @@ -106,6 +106,19 @@ spec: also grant access (in current kernels, writing the pid namespace's `pid_max`), so `capability` rules do not cover every `CAP_SYS_ADMIN` use. +## Using it outside this repository + +- The gadget and `../common` depend only on Inspektor Gadget, the libraries it + uses, and the `enforce_exec` types of the Inspektor Gadget fork. A guard test + (`common/enforcement/portability_test.go`) keeps it that way. +- A Go program drives it: `runtime.WithConfig` with a compiled policy, then + `runtime.GetBindings` to bind containers (see `runtime/runtime_kernel_test.go`). + There is no `ig` command for it yet; `ig run` alone fails because the policy + comes from the owner, not from gadget parameters. +- To move it into Inspektor Gadget: rewrite the import paths, drop the + per-gadget `Makefile` (its `REPO_ROOT` is this repository's depth) and add the + gadget to the central gadget list, which already sets `IG_SOURCE_PATH`. + ## Maps and events | Map | Content | @@ -115,7 +128,9 @@ spec: The `verdict` datasource shares `generation`, `cgroup`, `deny`, `mode` and `host_pid` with enforce_exec and adds `rule_id`, `origin_id`, `is_default`, -`comm`, `hook`, `syscall`, `flags`, `capability`, `fstype` and `slot`. Rule and origin +`comm`, `hook`, `syscall`, `flags`, `capability`, `fstype` and `slot`. When a +document's default deny strengthens another document's audit rule on the same +key, the event names the rule `default` with `is_default` false. Rule and origin names are in the owner's compiled policy (`RuleNames`, `Origins`). Reporting is limited to 100 events per CPU per second; enforcement is never throttled. diff --git a/pkg/ebpf/gadgets/enforce_privilege/examples/policy.yaml b/pkg/ebpf/gadgets/enforce_privilege/examples/policy.yaml new file mode 100644 index 0000000000..fff38d6dd3 --- /dev/null +++ b/pkg/ebpf/gadgets/enforce_privilege/examples/policy.yaml @@ -0,0 +1,28 @@ +# enforce_privilege policy document. One document per policy; the compiler can +# merge several documents into one slot (most limiting wins). +version: 1 +name: example +policy: + mode: enforce # audit | enforce | off + privilege: + default: allow # default for syscall and mountFsType targets + rules: + - name: no-new-namespaces + target: + syscall: [unshare, clone, clone3, setns] + flags: [CLONE_NEWUSER, CLONE_NEWNS] + action: deny + origin: example/namespaces + - name: no-proc-mount + target: + mountFsType: [proc, sysfs] + action: deny + - name: no-kernel-modules + target: + syscall: [init_module, finit_module] + action: deny + - name: watch-sys-admin + target: + capability: [SYS_ADMIN] + action: deny + mode: audit diff --git a/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml b/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml index 9024f4ffff..11eb6602bb 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml +++ b/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml @@ -1,6 +1,6 @@ name: enforce_privilege behavior: mutator -description: "Enforce SandboxPolicy privilege rules (capabilities, namespaces, mounts, ptrace, modules) for a live set of containers (BPF LSM)" +description: "Enforce privilege rules (capabilities, namespaces, mounts, ptrace, modules) for a live set of containers (BPF LSM)" annotations: enable-containers-datasource: "true" enforce-privilege: "true" diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/api.go b/pkg/ebpf/gadgets/enforce_privilege/policy/api.go deleted file mode 100644 index 0b6294b907..0000000000 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/api.go +++ /dev/null @@ -1,24 +0,0 @@ -package privpolicy - -import ( - "fmt" - - api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" -) - -// FromAPI converts a SandboxPolicy privilege section into a validated Document. -// Actor and path predicates are not supported yet and are rejected. -func FromAPI(name string, mode api.PolicyMode, section api.PrivilegeSection) (Document, error) { - doc := Document{Name: name, Version: 1, Policy: Policy{Mode: string(mode), Privilege: Section{Default: string(section.Default)}}} - for _, r := range section.Rules { - t := r.Target - if r.Actor != nil || len(t.Path) > 0 || len(t.PathClass) > 0 || t.FromImage != nil || len(t.FSType) > 0 { - return doc, fmt.Errorf("rule %q: actor and path targets are unsupported in the privilege section", r.Name) - } - doc.Policy.Privilege.Rules = append(doc.Policy.Privilege.Rules, Rule{ - Name: r.Name, Action: string(r.Action), Mode: string(r.Mode), Origin: r.Origin, - Target: Target{Syscall: t.Syscall, Flags: t.Flags, Capability: t.Capability, MountFSType: t.MountFSType}, - }) - } - return doc, doc.Validate() -} diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go index ff7d7b121e..60db0f2e7e 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go @@ -119,12 +119,14 @@ func CompileKernel(d Document, generation uint64) (Compiled, error) { } // CompileSlots compiles one node instance. Each slot holds one combination of -// policies, merged as the CRD design says for several policies matching one pod: +// documents, combined so that the most limiting result wins: // - rules are the union, each with its own name, origin and mode; the compiler // folds every rule on one key into the most limiting rank, so an explicit // deny beats an allow and an allow beats a default; -// - a default is deny if any policy's default is deny, else allow; its mode is -// the strictest among the policies whose default equals the merged one. +// - a default is deny if any document's default is deny, else allow; its mode +// is the strictest among the documents whose default equals the merged one; +// - an audit deny rule never weakens another document's enforce default deny +// (see the raise below), while an explicit allow still beats it. // // The default covers the syscall and mountFsType kinds only. Capability checks // have no default: only explicit capability rules apply, because capable() runs diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go index 55a97c12a3..d153fd10db 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go @@ -5,22 +5,12 @@ package privpolicy import ( "encoding/binary" + "os" "strings" "testing" - - "gopkg.in/yaml.v3" ) -// parse decodes a test policy with unknown fields rejected, then validates it. -func parse(y string) (Document, error) { - var d Document - dec := yaml.NewDecoder(strings.NewReader(y)) - dec.KnownFields(true) - if err := dec.Decode(&d); err != nil { - return d, err - } - return d, d.Validate() -} +func parse(y string) (Document, error) { return Parse(strings.NewReader(y)) } func mustParse(t *testing.T, y string) Document { t.Helper() @@ -248,3 +238,27 @@ func TestAuditDenyRaiseRespectsFlagKeys(t *testing.T) { t.Fatalf("unshare --net: rank %d, want enforce from B's default", r) } } + +func TestParse(t *testing.T) { + f, err := os.Open("../examples/policy.yaml") + if err != nil { + t.Fatal(err) + } + defer f.Close() + d, err := Parse(f) + if err != nil { + t.Fatalf("example: %v", err) + } + if _, err := CompileKernel(d, 1); err != nil { + t.Fatalf("example compile: %v", err) + } + for name, in := range map[string]string{ + "unknown field": "version: 1\npolicy: {mode: enforce, privilege: {default: allow}}\nbinding: {}\n", + "two documents": "version: 1\npolicy: {mode: enforce, privilege: {default: allow}}\n---\nversion: 1\n", + "too large": "version: 1\n#" + strings.Repeat("x", MaxBytes) + "\n", + } { + if _, err := parse(in); err == nil { + t.Errorf("%s: expected error", name) + } + } +} diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go b/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go index b44849ade6..d44d2dd39c 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go @@ -1,16 +1,24 @@ // Copyright 2026 The Kubescape Authors // SPDX-License-Identifier: Apache-2.0 -// Package privpolicy parses and compiles the privilege section of a sandbox -// policy into the enforce_privilege map entries. +// Package privpolicy defines the enforce_privilege policy document (YAML or +// JSON, see ../examples/policy.yaml), parses it, and compiles one or more +// documents into the gadget's map entries. package privpolicy import ( + "bytes" "fmt" + "io" "strings" + + "gopkg.in/yaml.v3" ) -const MaxRules = 128 +const ( + MaxBytes = 64 * 1024 + MaxRules = 128 +) type Document struct { // Name qualifies rule names in events when several policies merge. @@ -45,6 +53,28 @@ type Target struct { MountFSType []string `yaml:"mountFsType,omitempty" json:"mountFsType,omitempty"` } +// Parse reads one policy document (YAML or JSON) with unknown fields rejected. +func Parse(r io.Reader) (Document, error) { + var doc Document + data, err := io.ReadAll(io.LimitReader(r, MaxBytes+1)) + if err != nil { + return doc, err + } + if len(data) > MaxBytes { + return doc, fmt.Errorf("policy exceeds %d bytes", MaxBytes) + } + dec := yaml.NewDecoder(bytes.NewReader(data)) + dec.KnownFields(true) + if err := dec.Decode(&doc); err != nil { + return doc, fmt.Errorf("parsing policy: %w", err) + } + var extra any + if err := dec.Decode(&extra); err != io.EOF { + return doc, fmt.Errorf("parsing policy: one document per file") + } + return doc, doc.Validate() +} + func validMode(m string, allowEmpty bool) bool { return m == "audit" || m == "enforce" || (allowEmpty && m == "") } diff --git a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c index f1b04a7b6a..13d0ae97a4 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c +++ b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c @@ -1,7 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright 2026 The Kubescape Authors -// enforce_privilege: enforce the privilege section of a SandboxPolicy with BPF +// enforce_privilege: enforce a privilege policy (namespaces, mounts, +// capabilities, ptrace, modules and more) for a set of containers with BPF // LSM. The program is fixed; the policy is map data (privilege_policy.h), // frozen per instance; a rule change starts a replacement instance. Audit and // enforce share one code path: the rank of the deciding entry carries the mode. diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go new file mode 100644 index 0000000000..134d09f458 --- /dev/null +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go @@ -0,0 +1,139 @@ +//go:build linux && privilege_kernel + +// Copyright 2026 The Kubescape Authors +// SPDX-License-Identifier: Apache-2.0 + +package runtime + +// Opt-in test of the gadget with Inspektor Gadget alone: it parses the example +// policy file, starts the gadget from an OCI archive and binds one Docker +// container through this package's operator. Requires root, BPF LSM and a +// disposable container started with seccomp=unconfined and --cap-add SYS_ADMIN: +// +// PRIVILEGE_KERNEL_TEST=1 +// PRIVILEGE_IMAGE_ARCHIVE OCI archive from `ig image export` +// PRIVILEGE_IMAGE image name in the archive +// PRIVILEGE_CONTAINER_ID full Docker container ID +// PRIVILEGE_CGROUP_PATH its cgroup v2 path (relative to /sys/fs/cgroup) + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strconv" + "sync/atomic" + "testing" + "time" + + "github.com/inspektor-gadget/inspektor-gadget/pkg/datasource" + gadgetcontext "github.com/inspektor-gadget/inspektor-gadget/pkg/gadget-context" + "github.com/inspektor-gadget/inspektor-gadget/pkg/gadget-service/api" + apihelpers "github.com/inspektor-gadget/inspektor-gadget/pkg/gadget-service/api-helpers" + instancemanager "github.com/inspektor-gadget/inspektor-gadget/pkg/gadget-service/instance-manager" + "github.com/inspektor-gadget/inspektor-gadget/pkg/operators" + _ "github.com/inspektor-gadget/inspektor-gadget/pkg/operators/ebpf" + ocihandler "github.com/inspektor-gadget/inspektor-gadget/pkg/operators/oci-handler" + "github.com/inspektor-gadget/inspektor-gadget/pkg/operators/simple" + "github.com/inspektor-gadget/inspektor-gadget/pkg/runtime/local" + "golang.org/x/sys/unix" + orasoci "oras.land/oras-go/v2/content/oci" + + "github.com/kubescape/node-agent/pkg/ebpf/gadgets/common/enforcement" + privpolicy "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/policy" +) + +func TestGadgetWithInspektorGadgetAlone(t *testing.T) { + if os.Getenv("PRIVILEGE_KERNEL_TEST") != "1" { + t.Skip("explicit disposable kernel fixture opt-in required") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + f, err := os.Open("../examples/policy.yaml") + if err != nil { + t.Fatal(err) + } + doc, err := privpolicy.Parse(f) + f.Close() + if err != nil { + t.Fatal(err) + } + compiled, err := privpolicy.CompileKernel(doc, 1) + if err != nil { + t.Fatal(err) + } + id, cgroupPath := os.Getenv("PRIVILEGE_CONTAINER_ID"), os.Getenv("PRIVILEGE_CGROUP_PATH") + var st unix.Stat_t + if err := unix.Stat(filepath.Join("/sys/fs/cgroup", cgroupPath), &st); err != nil { + t.Fatal(err) + } + container := ContainerIdentity{Runtime: "docker", ID: id, CgroupID: st.Ino, CgroupPath: cgroupPath} + + store, err := orasoci.NewFromTar(ctx, os.Getenv("PRIVILEGE_IMAGE_ARCHIVE")) + if err != nil { + t.Fatal(err) + } + rt := local.New() + if err := rt.Init(nil); err != nil { + t.Fatal(err) + } + defer rt.Close() + manager, err := instancemanager.New(rt) + if err != nil { + t.Fatal(err) + } + handler := ocihandler.New() + params := apihelpers.ToParamDescs(handler.GlobalParams()).ToParams() + if err := params.Set("allow-enforcement", "true"); err != nil { + t.Fatal(err) + } + if err := handler.Init(params); err != nil { + t.Fatal(err) + } + var verdicts atomic.Int32 + var bindings *enforcement.Bindings + events := simple.New("privilege-test-verdicts", simple.WithPriority(100), simple.OnInit(func(gc operators.GadgetContext) error { + var err error + if bindings, err = GetBindings(gc); err != nil { + return err + } + return gc.GetDataSources()["verdict"].Subscribe(func(datasource.DataSource, datasource.Data) error { + verdicts.Add(1) + return nil + }, 100) + })) + runtimeMntns, err := enforcement.RuntimeMountNamespace() + if err != nil { + t.Fatal(err) + } + desired := &api.GadgetInstance{Id: "privilege-kernel-test", GadgetConfig: &api.GadgetRunRequest{Version: api.VersionGadgetRunProtocol, ImageName: os.Getenv("PRIVILEGE_IMAGE"), + ParamValues: map[string]string{"operator.oci.ebpf.host-mntns-id": strconv.FormatUint(runtimeMntns, 10)}}} + if err := manager.StartGadget(ctx, desired, gadgetcontext.WithOrasReadonlyTarget(store), + gadgetcontext.WithAdditionalDataOperators(handler, New(), events), + WithConfig(Config{Policy: compiled})); err != nil { + t.Fatal(err) + } + defer manager.StopGadget(context.Background(), desired.Id) + + unshare := func() error { return exec.CommandContext(ctx, "docker", "exec", id, "unshare", "-m", "true").Run() } + if err := unshare(); err != nil { + t.Fatalf("unbound container must not be judged: %v", err) + } + if err := bindings.Bind(enforcement.Binding{Container: container}); err != nil { + t.Fatal(err) + } + if unshare() == nil { + t.Fatal("the example policy denies unshare(CLONE_NEWNS) in a bound container") + } + if err := bindings.Unbind(container); err != nil { + t.Fatal(err) + } + if err := unshare(); err != nil { + t.Fatalf("unbound again must be allowed: %v", err) + } + time.Sleep(500 * time.Millisecond) + if verdicts.Load() == 0 { + t.Fatal("no verdict event for the deny") + } +} From c41c1441decfae6752c1eb7bd7c902a5d814962e Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 10:03:32 +0200 Subject: [PATCH 11/54] refactor(sandboxpolicy): translate SandboxPolicy privilege sections in the controller layer PrivilegeDocument (pkg/sandboxpolicy/privilege.go) replaces the gadget former FromAPI, so the gadget no longer imports the CRD types. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/compiler.go | 3 +-- pkg/sandboxpolicy/igbackend/privilege.go | 2 +- pkg/sandboxpolicy/privilege.go | 27 ++++++++++++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 pkg/sandboxpolicy/privilege.go diff --git a/pkg/sandboxpolicy/compiler.go b/pkg/sandboxpolicy/compiler.go index 1f69ffb23d..e7838e4e08 100644 --- a/pkg/sandboxpolicy/compiler.go +++ b/pkg/sandboxpolicy/compiler.go @@ -8,7 +8,6 @@ import ( "fmt" "sort" - privpolicy "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/policy" api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" ) @@ -172,7 +171,7 @@ func validateShape(spec api.PolicySpec) error { return fmt.Errorf("file, network and identity sections are unsupported") } if spec.Privilege != nil { - if _, err := privpolicy.FromAPI("", spec.Mode, *spec.Privilege); err != nil { + if _, err := PrivilegeDocument("", spec.Mode, *spec.Privilege); err != nil { return fmt.Errorf("privilege: %w", err) } } diff --git a/pkg/sandboxpolicy/igbackend/privilege.go b/pkg/sandboxpolicy/igbackend/privilege.go index d85e24972b..ce1088a970 100644 --- a/pkg/sandboxpolicy/igbackend/privilege.go +++ b/pkg/sandboxpolicy/igbackend/privilege.go @@ -309,7 +309,7 @@ func (b *PrivilegeBackend) replaceInstance(ctx context.Context, prepared, extra for i, c := range keys { slots[c] = uint32(i) for _, p := range wanted[c] { - doc, err := privpolicy.FromAPI(p.Identity.Namespace+"/"+p.Identity.Name, p.Mode, *p.Privilege) + doc, err := sandboxpolicy.PrivilegeDocument(p.Identity.Namespace+"/"+p.Identity.Name, p.Mode, *p.Privilege) if err != nil { return err } diff --git a/pkg/sandboxpolicy/privilege.go b/pkg/sandboxpolicy/privilege.go new file mode 100644 index 0000000000..42e2e8e7a1 --- /dev/null +++ b/pkg/sandboxpolicy/privilege.go @@ -0,0 +1,27 @@ +package sandboxpolicy + +import ( + "fmt" + + privpolicy "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/policy" + api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" +) + +// PrivilegeDocument translates the privilege section of one SandboxPolicy into +// the privilege gadget's own policy document. name qualifies rule names in +// events. Actor and path predicates are not supported by the gadget yet and +// are rejected. +func PrivilegeDocument(name string, mode api.PolicyMode, section api.PrivilegeSection) (privpolicy.Document, error) { + doc := privpolicy.Document{Name: name, Version: 1, Policy: privpolicy.Policy{Mode: string(mode), Privilege: privpolicy.Section{Default: string(section.Default)}}} + for _, r := range section.Rules { + t := r.Target + if r.Actor != nil || len(t.Path) > 0 || len(t.PathClass) > 0 || t.FromImage != nil || len(t.FSType) > 0 { + return doc, fmt.Errorf("rule %q: actor and path targets are unsupported in the privilege section", r.Name) + } + doc.Policy.Privilege.Rules = append(doc.Policy.Privilege.Rules, privpolicy.Rule{ + Name: r.Name, Action: string(r.Action), Mode: string(r.Mode), Origin: r.Origin, + Target: privpolicy.Target{Syscall: t.Syscall, Flags: t.Flags, Capability: t.Capability, MountFSType: t.MountFSType}, + }) + } + return doc, doc.Validate() +} From f27230366f95e5dc0150474d27a75cda1dee4b86 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 10:21:57 +0200 Subject: [PATCH 12/54] fix(sandboxpolicy): refuse privilege enforcement when BPF LSM is not active Without "bpf" in the host's active LSM list, BPF LSM programs attach but never run (for example default AKS Ubuntu and Azure Linux images), so a policy could report Ready with no enforcement. The service now reads /sys/kernel/security/lsm at startup; if bpf is missing it creates no privilege backend, privilege bindings are refused, and every policy with a privilege section reports Ready=False with reason BPFLSMInactive. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/availability.go | 39 +++++++++++++++ pkg/sandboxpolicy/igbackend/lsm.go | 34 +++++++++++++ pkg/sandboxpolicy/igbackend/lsm_test.go | 37 ++++++++++++++ .../igbackend/privilege_kernel_test.go | 3 ++ pkg/sandboxpolicy/integration/service.go | 6 ++- pkg/sandboxpolicy/sections.go | 23 +++++++-- pkg/sandboxpolicy/sections_test.go | 50 ++++++++++++++----- pkg/sandboxpolicy/status.go | 9 ++++ 8 files changed, 183 insertions(+), 18 deletions(-) create mode 100644 pkg/sandboxpolicy/availability.go create mode 100644 pkg/sandboxpolicy/igbackend/lsm.go create mode 100644 pkg/sandboxpolicy/igbackend/lsm_test.go diff --git a/pkg/sandboxpolicy/availability.go b/pkg/sandboxpolicy/availability.go new file mode 100644 index 0000000000..073ac7f979 --- /dev/null +++ b/pkg/sandboxpolicy/availability.go @@ -0,0 +1,39 @@ +package sandboxpolicy + +import ( + "errors" + "fmt" +) + +// EnforcementUnavailableError means a policy section cannot be enforced on this +// node. Reason becomes the Ready condition reason in the node status. +type EnforcementUnavailableError struct { + Section, Reason, Detail string +} + +func (e *EnforcementUnavailableError) Error() string { + return fmt.Sprintf("%s enforcement is unavailable on this node: %s", e.Section, e.Detail) +} + +// PrivilegeAvailability is implemented by a backend that knows whether the +// privilege section can be enforced on this node. +type PrivilegeAvailability interface { + PrivilegeUnavailable() error +} + +// privilegeUnavailable returns why the privilege section cannot be enforced, +// or nil. It is called with c.mu held. +func (c *Controller) privilegeUnavailable() error { + if a, ok := c.backend.(PrivilegeAvailability); ok { + return a.PrivilegeUnavailable() + } + return nil +} + +func unavailableReason(err error) (string, string, bool) { + var u *EnforcementUnavailableError + if !errors.As(err, &u) { + return "", "", false + } + return u.Reason, u.Error(), true +} diff --git a/pkg/sandboxpolicy/igbackend/lsm.go b/pkg/sandboxpolicy/igbackend/lsm.go new file mode 100644 index 0000000000..a5ac7e8867 --- /dev/null +++ b/pkg/sandboxpolicy/igbackend/lsm.go @@ -0,0 +1,34 @@ +package igbackend + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/kubescape/node-agent/pkg/sandboxpolicy" +) + +// CheckBPFLSM returns an EnforcementUnavailableError unless "bpf" is in the +// host's active LSM list. Without it, BPF LSM programs attach but never run +// (for example the default AKS Ubuntu and Azure Linux images), so enforcement +// would be reported while nothing is enforced. +func CheckBPFLSM(hostRoot string) error { + path := filepath.Join(hostRoot, "sys/kernel/security/lsm") + data, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return &sandboxpolicy.EnforcementUnavailableError{Section: "privilege", Reason: "BPFLSMInactive", Detail: fmt.Sprintf("%s is missing (securityfs not mounted)", path)} + } + if err != nil { + return &sandboxpolicy.EnforcementUnavailableError{Section: "privilege", Reason: "BPFLSMInactive", Detail: fmt.Sprintf("reading %s: %v", path, err)} + } + for _, lsm := range strings.Split(strings.TrimSpace(string(data)), ",") { + if lsm == "bpf" { + return nil + } + } + return &sandboxpolicy.EnforcementUnavailableError{Section: "privilege", Reason: "BPFLSMInactive", + Detail: fmt.Sprintf("BPF LSM is not active (active LSMs: %s); add bpf to the lsm= kernel parameter", strings.TrimSpace(string(data)))} +} diff --git a/pkg/sandboxpolicy/igbackend/lsm_test.go b/pkg/sandboxpolicy/igbackend/lsm_test.go new file mode 100644 index 0000000000..4a2a4fec2b --- /dev/null +++ b/pkg/sandboxpolicy/igbackend/lsm_test.go @@ -0,0 +1,37 @@ +package igbackend + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/kubescape/node-agent/pkg/sandboxpolicy" +) + +func TestCheckBPFLSM(t *testing.T) { + write := func(list string) string { + root := t.TempDir() + dir := filepath.Join(root, "sys/kernel/security") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "lsm"), []byte(list), 0o644); err != nil { + t.Fatal(err) + } + return root + } + if err := CheckBPFLSM(write("lockdown,capability,landlock,yama,apparmor,bpf\n")); err != nil { + t.Fatalf("bpf active: %v", err) + } + for name, root := range map[string]string{ + "no bpf": write("lockdown,capability,landlock,yama,apparmor\n"), + "bpf-like": write("lockdown,capability,bpfx\n"), + "no securityfs": t.TempDir(), + } { + var u *sandboxpolicy.EnforcementUnavailableError + if err := CheckBPFLSM(root); !errors.As(err, &u) || u.Reason != "BPFLSMInactive" { + t.Errorf("%s: %v", name, err) + } + } +} diff --git a/pkg/sandboxpolicy/igbackend/privilege_kernel_test.go b/pkg/sandboxpolicy/igbackend/privilege_kernel_test.go index 3f552cb629..8a4298e332 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_kernel_test.go +++ b/pkg/sandboxpolicy/igbackend/privilege_kernel_test.go @@ -40,6 +40,9 @@ func TestPrivilegeBackendKernel(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() + if err := CheckBPFLSM("/"); err != nil { + t.Fatalf("this test needs an active BPF LSM: %v", err) + } bindingOf := func(id, cgroupPath string) sandboxpolicy.Binding { var st unix.Stat_t if err := unix.Stat(filepath.Join("/sys/fs/cgroup", cgroupPath), &st); err != nil { diff --git a/pkg/sandboxpolicy/integration/service.go b/pkg/sandboxpolicy/integration/service.go index 51344e171e..94a1cc9c06 100644 --- a/pkg/sandboxpolicy/integration/service.go +++ b/pkg/sandboxpolicy/integration/service.go @@ -135,7 +135,11 @@ func NewService(ctx context.Context, options ServiceOptions) (*Service, error) { return nil, err } var privilege sandboxpolicy.Backend + var privilegeUnavailable error if options.PrivilegeImageName != "" { + privilegeUnavailable = igbackend.CheckBPFLSM(host.HostRoot) + } + if options.PrivilegeImageName != "" && privilegeUnavailable == nil { privilegeEngine, err := igbackend.NewNativePrivilegeEngine(r, kubemanager.NewKubeManager(options.Containers, options.Tracers), store, options.PrivilegeImageName) if err == nil { privilege, err = igbackend.NewPrivilege(privilegeEngine, igbackend.LinuxValidator{ProcRoot: host.HostProcFs, CgroupRoot: filepath.Join(host.HostRoot, "sys/fs/cgroup")}, options.NodeName) @@ -146,7 +150,7 @@ func NewService(ctx context.Context, options ServiceOptions) (*Service, error) { return nil, err } } - s.backend, err = sandboxpolicy.NewSectionBackend(full, privilege) + s.backend, err = sandboxpolicy.NewSectionBackend(full, privilege, privilegeUnavailable) if err != nil { _ = full.Close(ctx) _ = s.closeRuntime() diff --git a/pkg/sandboxpolicy/sections.go b/pkg/sandboxpolicy/sections.go index d73bbae149..9f1432cc1c 100644 --- a/pkg/sandboxpolicy/sections.go +++ b/pkg/sandboxpolicy/sections.go @@ -24,8 +24,10 @@ import ( type SectionBackend struct { mu sync.Mutex exec Backend - privilege Backend // nil: privilege sections are rejected - active map[string]sectionPlans + privilege Backend // nil: privilege sections are rejected with unavailable + // unavailable says why privilege sections are rejected (privilege == nil). + unavailable error + active map[string]sectionPlans } type sectionPlans struct { @@ -41,13 +43,24 @@ var ( _ ProtectionAttester = (*SectionBackend)(nil) ) -func NewSectionBackend(exec, privilege Backend) (*SectionBackend, error) { +// NewSectionBackend combines the section backends. When privilege is nil, +// unavailable says why (nil means no privilege image is configured). +func NewSectionBackend(exec, privilege Backend, unavailable error) (*SectionBackend, error) { if exec == nil { return nil, fmt.Errorf("exec backend is required") } - return &SectionBackend{exec: exec, privilege: privilege, active: map[string]sectionPlans{}}, nil + if privilege == nil && unavailable == nil { + unavailable = &EnforcementUnavailableError{Section: "privilege", Reason: "PrivilegeNotConfigured", Detail: "no privilege gadget image is configured"} + } + if privilege != nil { + unavailable = nil + } + return &SectionBackend{exec: exec, privilege: privilege, unavailable: unavailable, active: map[string]sectionPlans{}}, nil } +// PrivilegeUnavailable returns why privilege sections cannot be enforced. +func (s *SectionBackend) PrivilegeUnavailable() error { return s.unavailable } + // execOnly reports whether the plans need only the exec backend. func (p sectionPlans) execOnly() bool { return p.privilege.Revision == "" && !p.privilegeCleanup } @@ -158,7 +171,7 @@ func (s *SectionBackend) Replace(ctx context.Context, binding Binding, plan Plan return r, err } if s.privilege == nil { - return ReplaceResult{}, fmt.Errorf("privilege sections are unsupported: no privilege image configured") + return ReplaceResult{}, s.unavailable } e := apply(ctx, s.exec, binding, old.exec, want.exec) diff --git a/pkg/sandboxpolicy/sections_test.go b/pkg/sandboxpolicy/sections_test.go index 33c407baa0..cf252abfa5 100644 --- a/pkg/sandboxpolicy/sections_test.go +++ b/pkg/sandboxpolicy/sections_test.go @@ -2,9 +2,11 @@ package sandboxpolicy import ( "context" + "errors" "testing" api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func privilegeSection(def api.Action) *api.PrivilegeSection { @@ -17,7 +19,7 @@ func sectionPlan(revision string, policies ...CompiledPolicy) Plan { func TestSectionBackendExecOnlyPlanIsUnchanged(t *testing.T) { exec, privilege := &fakeBackend{plans: map[string]Plan{}}, &fakeBackend{plans: map[string]Plan{}} - s, err := NewSectionBackend(exec, privilege) + s, err := NewSectionBackend(exec, privilege, nil) if err != nil { t.Fatal(err) } @@ -39,7 +41,7 @@ func TestSectionBackendExecOnlyPlanIsUnchanged(t *testing.T) { func TestSectionBackendSplitsAndKeepsUnchangedSection(t *testing.T) { exec, privilege := &fakeBackend{plans: map[string]Plan{}}, &fakeBackend{plans: map[string]Plan{}} - s, _ := NewSectionBackend(exec, privilege) + s, _ := NewSectionBackend(exec, privilege, nil) b := Binding{ContainerID: "c"} execPolicy := CompiledPolicy{Revision: "e", Exec: api.ExecSection{Default: api.ActionDeny}} privPolicy := CompiledPolicy{Revision: "p", Privilege: privilegeSection(api.ActionAllow)} @@ -72,7 +74,7 @@ func TestSectionBackendSplitsAndKeepsUnchangedSection(t *testing.T) { func TestSectionBackendPrivilegeFailureRestoresExec(t *testing.T) { exec, privilege := &fakeBackend{plans: map[string]Plan{}}, &fakeBackend{plans: map[string]Plan{}, fail: true} - s, _ := NewSectionBackend(exec, privilege) + s, _ := NewSectionBackend(exec, privilege, nil) b := Binding{ContainerID: "c"} execPolicy := CompiledPolicy{Revision: "e", Exec: api.ExecSection{Default: api.ActionDeny}} r, err := s.Replace(context.Background(), b, sectionPlan("r1", execPolicy, CompiledPolicy{Revision: "p", Privilege: privilegeSection(api.ActionAllow)})) @@ -85,7 +87,7 @@ func TestSectionBackendPrivilegeFailureRestoresExec(t *testing.T) { } func TestSectionBackendRejectsPrivilegeWithoutImage(t *testing.T) { - s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, nil) + s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, nil, nil) if _, err := s.Replace(context.Background(), Binding{}, sectionPlan("r", CompiledPolicy{Privilege: privilegeSection(api.ActionDeny)})); err == nil { t.Fatal("privilege must be rejected without a privilege backend") } @@ -107,7 +109,7 @@ func TestCompileKeepsPrivilegeSection(t *testing.T) { func TestSectionBackendKeepsCommittedCleanupErrors(t *testing.T) { // Exec-only: the exec backend result passes through unchanged. exec := &fakeBackend{plans: map[string]Plan{}, postCommit: true} - s, _ := NewSectionBackend(exec, &fakeBackend{plans: map[string]Plan{}}) + s, _ := NewSectionBackend(exec, &fakeBackend{plans: map[string]Plan{}}, nil) b := Binding{ContainerID: "c"} r, err := s.Replace(context.Background(), b, sectionPlan("r1", CompiledPolicy{Exec: api.ExecSection{Default: api.ActionDeny}})) if err == nil || !r.Committed || r.ActiveRevision != "r1" { @@ -115,7 +117,7 @@ func TestSectionBackendKeepsCommittedCleanupErrors(t *testing.T) { } // With privilege: a committed section with a cleanup error does not roll back. privilege := &fakeBackend{plans: map[string]Plan{}, postCommit: true} - s, _ = NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege) + s, _ = NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege, nil) r, err = s.Replace(context.Background(), b, sectionPlan("r2", CompiledPolicy{Privilege: privilegeSection(api.ActionAllow)})) if err == nil || !r.Committed || r.ActiveRevision != "r2" { t.Fatalf("privilege cleanup error must stay committed: %+v %v", r, err) @@ -124,7 +126,7 @@ func TestSectionBackendKeepsCommittedCleanupErrors(t *testing.T) { func TestSectionBackendPrivilegeChangeKeepsExecRevision(t *testing.T) { exec, privilege := &fakeBackend{plans: map[string]Plan{}}, &fakeBackend{plans: map[string]Plan{}} - s, _ := NewSectionBackend(exec, privilege) + s, _ := NewSectionBackend(exec, privilege, nil) b := Binding{ContainerID: "c"} p := CompiledPolicy{Revision: "v1", Exec: api.ExecSection{Default: api.ActionDeny}, Privilege: privilegeSection(api.ActionAllow)} if _, err := s.Replace(context.Background(), b, sectionPlan("r1", p)); err != nil { @@ -142,7 +144,7 @@ func TestSectionBackendPrivilegeChangeKeepsExecRevision(t *testing.T) { func TestSectionBackendSendsAllPrivilegePolicies(t *testing.T) { privilege := &fakeBackend{plans: map[string]Plan{}} - s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege) + s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege, nil) b := Binding{ContainerID: "c"} p1 := CompiledPolicy{Identity: PolicyIdentity{UID: "1"}, Privilege: privilegeSection(api.ActionAllow)} p2 := CompiledPolicy{Identity: PolicyIdentity{UID: "2"}, Privilege: privilegeSection(api.ActionDeny)} @@ -166,7 +168,7 @@ func (p *preparingBackend) Prepare(_ context.Context, plans []Plan) error { func TestSectionBackendPassesPrepare(t *testing.T) { privilege := &preparingBackend{fakeBackend: fakeBackend{plans: map[string]Plan{}}} - s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege) + s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege, nil) if err := s.Prepare(context.Background(), []Plan{sectionPlan("r", CompiledPolicy{Privilege: privilegeSection(api.ActionDeny)})}); err != nil || len(privilege.prepared) != 1 { t.Fatalf("prepare: %v %v", err, privilege.prepared) } @@ -174,7 +176,7 @@ func TestSectionBackendPassesPrepare(t *testing.T) { func TestSectionBackendModeChangeReachesPrivilege(t *testing.T) { privilege := &fakeBackend{plans: map[string]Plan{}} - s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege) + s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege, nil) b := Binding{ContainerID: "c"} p := CompiledPolicy{Mode: api.ModeAudit, Privilege: privilegeSection(api.ActionAllow)} if _, err := s.Replace(context.Background(), b, sectionPlan("r1", p)); err != nil { @@ -191,7 +193,7 @@ func TestSectionBackendModeChangeReachesPrivilege(t *testing.T) { func TestSectionBackendRetriesPrivilegeCleanup(t *testing.T) { privilege := &fakeBackend{plans: map[string]Plan{}} - s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege) + s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege, nil) b := Binding{ContainerID: "c"} if _, err := s.Replace(context.Background(), b, sectionPlan("r1", CompiledPolicy{Privilege: privilegeSection(api.ActionAllow)})); err != nil { t.Fatal(err) @@ -212,7 +214,7 @@ func TestSectionBackendRetriesPrivilegeCleanup(t *testing.T) { func TestSectionBackendPartialWithdrawIsUnconfirmed(t *testing.T) { privilege := &fakeBackend{plans: map[string]Plan{}} - s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege) + s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege, nil) b := Binding{ContainerID: "c"} p := CompiledPolicy{Exec: api.ExecSection{Default: api.ActionDeny}, Privilege: privilegeSection(api.ActionAllow)} if _, err := s.Replace(context.Background(), b, sectionPlan("r1", p)); err != nil { @@ -246,3 +248,27 @@ func TestControllerPrepareScheduled(t *testing.T) { t.Fatalf("prepared %+v", preparer.prepared) } } + +func TestPrivilegeUnavailableIsReportedAndRefused(t *testing.T) { + ctx := context.Background() + p := policy("with-privilege", "ns") + p.Spec.Privilege = privilegeSection(api.ActionDeny) + c, _, w, _ := setup(t, p) + lsm := &EnforcementUnavailableError{Section: "privilege", Reason: "BPFLSMInactive", Detail: "BPF LSM is not active"} + sections, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, nil, lsm) + c.backend = sections + if err := c.InitialSync(ctx); err != nil && !errors.Is(err, lsm) { + t.Logf("initial sync: %v", err) + } + report := w.reports[string(p.UID)] + if report == nil || len(report.Status.Conditions) == 0 { + t.Fatal("no status report") + } + ready := report.Status.Conditions[0] + if ready.Status != metav1.ConditionFalse || ready.Reason != "BPFLSMInactive" { + t.Fatalf("ready condition %+v", ready) + } + if _, err := sections.Replace(ctx, Binding{ContainerID: "c"}, sectionPlan("r", CompiledPolicy{Privilege: privilegeSection(api.ActionDeny)})); !errors.Is(err, lsm) { + t.Fatalf("privilege binding must be refused: %v", err) + } +} diff --git a/pkg/sandboxpolicy/status.go b/pkg/sandboxpolicy/status.go index bc7536faaf..14e6db3097 100644 --- a/pkg/sandboxpolicy/status.go +++ b/pkg/sandboxpolicy/status.go @@ -53,6 +53,12 @@ func (c *Controller) report(p desiredPolicy, failure error) *api.SandboxPolicyNo if p.spec.Mode == api.ModeOff { message = "Policy is off" } + if p.spec.Privilege != nil && p.spec.Mode != api.ModeOff { + // Without BPF LSM the programs attach but never run: never report Ready. + if err := c.privilegeUnavailable(); err != nil { + failure = errors.Join(failure, err) + } + } var uncertain *protectionError unconfirmed = unconfirmed || errors.As(failure, &uncertain) if unconfirmed { @@ -78,6 +84,9 @@ func (c *Controller) report(p desiredPolicy, failure error) *api.SandboxPolicyNo message = "Binding withdrawal committed; enforcement removed for that binding, resource cleanup pending" stale = generationStale } + if r, m, ok := unavailableReason(failure); ok { + reason, message = r, m + } if unconfirmed { reason = "ProtectionUnconfirmed" message = "Owned binding protection cannot be confirmed; workload may be unprotected; ownership retained only for recovery/cleanup" From ed1ee7cafcab236988eb8c04d4a0ea7e1d2fbecd Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 12:03:53 +0200 Subject: [PATCH 13/54] docs(enforce_privilege): mountFsType does not cover bind mounts Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/enforce_privilege/README.md | 3 +++ pkg/ebpf/gadgets/enforce_privilege/examples/policy.yaml | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index bb1e844fa0..920bdb2658 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -61,6 +61,9 @@ policy: independent of rule order. Each rule has its own mode. Each namespace flag of a call resolves on its own. - Capability checks have no default; only explicit capability rules apply. +- `mountFsType` judges the creation of a new filesystem instance (a new + superblock, from `mount(2)` or the new mount API). Bind mounts and remounts + of an existing one are judged only by the `mount` syscall rule. - A `default: deny` also denies `setuid`/`setgid` (package managers drop privileges), implicit kernel module loads (judged as `init_module`, for example an `iptables` match module) and new superblocks of any filesystem diff --git a/pkg/ebpf/gadgets/enforce_privilege/examples/policy.yaml b/pkg/ebpf/gadgets/enforce_privilege/examples/policy.yaml index fff38d6dd3..8be030b89a 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/examples/policy.yaml +++ b/pkg/ebpf/gadgets/enforce_privilege/examples/policy.yaml @@ -13,8 +13,8 @@ policy: flags: [CLONE_NEWUSER, CLONE_NEWNS] action: deny origin: example/namespaces - - name: no-proc-mount - target: + - name: no-proc-mount # new proc/sysfs mounts only; bind mounts + target: # are judged by the `mount` syscall rule mountFsType: [proc, sysfs] action: deny - name: no-kernel-modules From 09b5fdc4472bc7eac7e2427ca9b1259ca8bfa503 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:05:46 +0200 Subject: [PATCH 14/54] test(enforcement): only ../common/ includes may leave the gadget folder Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- .../common/enforcement/portability_test.go | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/pkg/ebpf/gadgets/common/enforcement/portability_test.go b/pkg/ebpf/gadgets/common/enforcement/portability_test.go index 7b5dee9477..2a0e9a17c9 100644 --- a/pkg/ebpf/gadgets/common/enforcement/portability_test.go +++ b/pkg/ebpf/gadgets/common/enforcement/portability_test.go @@ -38,8 +38,18 @@ var allowedImports = []string{ // Names of the host product and its policy API, which the gadgets must not use. var forbiddenWords = regexp.MustCompile(`(?i)sandbox[ _-]?polic|kubescape\.io|\barmo(sec)?\b|node-agent|kubescape/storage|\bkubernetes\b|\bpods?\b`) -// A BPF include must stay inside the gadget folders: "x.h" or "../common/x.h". -var badInclude = regexp.MustCompile(`#include\s+"(\.\./){2,}|#include\s+"\.\./[^c]`) +// A quoted BPF include must stay inside the gadget folders: "x.h" or +// "../common/x.h" (checked by includeOutside). +var quotedInclude = regexp.MustCompile(`#include\s+"([^"]*)"`) + +func includeOutside(line string) bool { + m := quotedInclude.FindStringSubmatch(line) + if m == nil || !strings.HasPrefix(m[1], "../") { + return false + } + rest, ok := strings.CutPrefix(m[1], "../common/") + return !ok || strings.Contains(rest, "..") +} func walkGadgetFiles(t *testing.T, fn func(path string, data []byte)) { t.Helper() @@ -105,7 +115,7 @@ func TestGadgetFilesUseNeutralNames(t *testing.T) { strings.HasPrefix(trimmed, "// Copyright") || strings.HasPrefix(trimmed, "# Copyright") { continue // import paths are rewritten on a move; license headers stay } - if badInclude.MatchString(line) { + if (strings.HasSuffix(path, ".c") || strings.HasSuffix(path, ".h")) && includeOutside(line) { t.Errorf("%s:%d includes a header outside the gadget folders", path, i+1) } if m := forbiddenWords.FindString(line); m != "" { @@ -114,3 +124,18 @@ func TestGadgetFilesUseNeutralNames(t *testing.T) { } }) } + +func TestIncludeOutside(t *testing.T) { + for line, want := range map[string]bool{ + `#include "privilege_policy.h"`: false, + `#include "../common/bindings.h"`: false, + `#include `: false, + `#include "../config/x.h"`: true, + `#include "../../pkg/x.h"`: true, + `#include "../common/../../pkg/x.h"`: true, + } { + if got := includeOutside(line); got != want { + t.Errorf("%s: got %v, want %v", line, got, want) + } + } +} From 128a0142825ba39902baf81d2e18a1a77f2bce82 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:05:47 +0200 Subject: [PATCH 15/54] test(sandboxpolicy): fail on an unexpected sync error in the LSM test Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/sections_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sandboxpolicy/sections_test.go b/pkg/sandboxpolicy/sections_test.go index cf252abfa5..7b000f1d2c 100644 --- a/pkg/sandboxpolicy/sections_test.go +++ b/pkg/sandboxpolicy/sections_test.go @@ -258,7 +258,7 @@ func TestPrivilegeUnavailableIsReportedAndRefused(t *testing.T) { sections, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, nil, lsm) c.backend = sections if err := c.InitialSync(ctx); err != nil && !errors.Is(err, lsm) { - t.Logf("initial sync: %v", err) + t.Fatalf("initial sync: %v", err) } report := w.reports[string(p.UID)] if report == nil || len(report.Status.Conditions) == 0 { From da814120f1555390ec509b50f71a2394247df3e3 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:06:58 +0200 Subject: [PATCH 16/54] test(enforce_privilege): the kernel test takes an absolute cgroup v2 path Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- .../gadgets/enforce_privilege/runtime/runtime_kernel_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go index 134d09f458..4dede884d2 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go @@ -14,7 +14,7 @@ package runtime // PRIVILEGE_IMAGE_ARCHIVE OCI archive from `ig image export` // PRIVILEGE_IMAGE image name in the archive // PRIVILEGE_CONTAINER_ID full Docker container ID -// PRIVILEGE_CGROUP_PATH its cgroup v2 path (relative to /sys/fs/cgroup) +// PRIVILEGE_CGROUP_PATH its cgroup v2 path under /sys/fs/cgroup, with a leading "/" (as in /proc//cgroup) import ( "context" From d0df3d45fe4ded4e3c00bfe277172145bf2560c5 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:06:58 +0200 Subject: [PATCH 17/54] test(sandboxpolicy): the kernel test takes an absolute cgroup v2 path Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/igbackend/privilege_kernel_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sandboxpolicy/igbackend/privilege_kernel_test.go b/pkg/sandboxpolicy/igbackend/privilege_kernel_test.go index 8a4298e332..98e6a7076c 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_kernel_test.go +++ b/pkg/sandboxpolicy/igbackend/privilege_kernel_test.go @@ -10,7 +10,7 @@ package igbackend // IG_PRIVILEGE_IMAGE_ARCHIVE OCI archive from `ig image export` // IG_PRIVILEGE_IMAGE image name in the archive // IG_PRIVILEGE_CONTAINER_ID full Docker container ID -// IG_PRIVILEGE_CGROUP_PATH its cgroup v2 path (relative to /sys/fs/cgroup) +// IG_PRIVILEGE_CGROUP_PATH its cgroup v2 path under /sys/fs/cgroup, with a leading "/" (as in /proc//cgroup) // IG_PRIVILEGE_CONTAINER_ID2, IG_PRIVILEGE_CGROUP_PATH2 a second container // // With two containers it also checks the node-wide instance: the second From ebb52fdb99d0038f8f2e6d7474a9e7d621d45cb5 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:04:43 +0200 Subject: [PATCH 18/54] test(enforcement): include guard also catches spaced, absolute and nested upward includes Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- .../common/enforcement/portability_test.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/pkg/ebpf/gadgets/common/enforcement/portability_test.go b/pkg/ebpf/gadgets/common/enforcement/portability_test.go index 2a0e9a17c9..298709b2f2 100644 --- a/pkg/ebpf/gadgets/common/enforcement/portability_test.go +++ b/pkg/ebpf/gadgets/common/enforcement/portability_test.go @@ -40,15 +40,24 @@ var forbiddenWords = regexp.MustCompile(`(?i)sandbox[ _-]?polic|kubescape\.io|\b // A quoted BPF include must stay inside the gadget folders: "x.h" or // "../common/x.h" (checked by includeOutside). -var quotedInclude = regexp.MustCompile(`#include\s+"([^"]*)"`) +var quotedInclude = regexp.MustCompile(`#\s*include\s*"([^"]*)"`) func includeOutside(line string) bool { m := quotedInclude.FindStringSubmatch(line) - if m == nil || !strings.HasPrefix(m[1], "../") { + if m == nil { return false } + if strings.HasPrefix(m[1], "/") { + return true // absolute + } rest, ok := strings.CutPrefix(m[1], "../common/") - return !ok || strings.Contains(rest, "..") + if !ok { + rest = m[1] + if strings.HasPrefix(rest, "../") { + return true + } + } + return strings.Contains(rest, "..") } func walkGadgetFiles(t *testing.T, fn func(path string, data []byte)) { @@ -133,6 +142,9 @@ func TestIncludeOutside(t *testing.T) { `#include "../config/x.h"`: true, `#include "../../pkg/x.h"`: true, `#include "../common/../../pkg/x.h"`: true, + `# include "../../pkg/x.h"`: true, + `#include "/usr/include/x.h"`: true, + `#include "sub/../../x.h"`: true, } { if got := includeOutside(line); got != want { t.Errorf("%s: got %v, want %v", line, got, want) From 32f5277b8fcc794e61a5bcada2ee4144921a31c9 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:19:44 +0200 Subject: [PATCH 19/54] fix(enforce_privilege): refuse a fork whose child cannot be armed in an enforcing slot When an armed task forks and the child's task storage mark cannot be allocated, the child escaped every rule while the instance stayed ready. Now an enforcing slot (any enforce entry) refuses the fork with -ENOMEM (a clone into another cgroup too), and counts a tracking fault per slot in arming_faults. A lost mark at exec, which cannot be refused, is counted the same way. Audit-only slots and unbound cgroups allow the fork. A new thread of an armed group leader is covered through the leader and is never refused. Moving a container to another slot arms its running tasks again. The binding controller reads the count with SlotFaults. A test-only parameter injects the failure for the new kernel test. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/common/arming.h | 60 +++++++- .../gadgets/common/enforcement/bindings.go | 60 +++++++- .../common/enforcement/bindings_test.go | 48 ++++++- .../gadgets/enforce_privilege/gadget.yaml | 4 + .../enforce_privilege/policy/kernel.go | 9 +- .../enforce_privilege/policy/kernel_test.go | 3 +- .../gadgets/enforce_privilege/program.bpf.c | 46 +++++- .../enforce_privilege/runtime/runtime.go | 1 + .../runtime/runtime_kernel_test.go | 134 ++++++++++++++---- 9 files changed, 311 insertions(+), 54 deletions(-) diff --git a/pkg/ebpf/gadgets/common/arming.h b/pkg/ebpf/gadgets/common/arming.h index d845e3d08a..029a56a216 100644 --- a/pkg/ebpf/gadgets/common/arming.h +++ b/pkg/ebpf/gadgets/common/arming.h @@ -53,8 +53,35 @@ static __always_inline bool arming_task_armed(struct task_struct *task) return a && *a; } +// Fault injection for tests only: fail every child mark at fork. Owners never +// set it in production. +const volatile bool arming_fail_fork_marks = false; + +// Tracking faults per policy slot, for the owner's health check: a mark that +// failed for a task in a bound cgroup whose slot enforces. The array is +// preallocated, so counting never fails. A new instance starts at zero and +// arms the running tasks again. +#ifndef ARMING_FAULT_SLOTS +#define ARMING_FAULT_SLOTS 64 +#endif + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, ARMING_FAULT_SLOTS); + __type(key, __u32); + __type(value, __u64); +} arming_faults SEC(".maps"); + +static __always_inline void arming_slot_fault(__u32 slot) +{ + __u64 *value = bpf_map_lookup_elem(&arming_faults, &slot); + + if (value) + __sync_fetch_and_add(value, 1); +} + // arming_mark returns false when the task storage could not be allocated (memory -// pressure): the task then stays unarmed, and the caller counts the failure. +// pressure): the task then stays unarmed, and the caller handles the failure. static __always_inline bool arming_mark(struct task_struct *task) { __u32 *a = bpf_task_storage_get(&armed_tasks, task, 0, @@ -113,12 +140,35 @@ static __always_inline bool arming_on_exec(void) return arming_mark(task); } +#define ARMING_CLONE_THREAD 0x00010000ULL + // arming_on_fork runs in task_alloc: the child inherits the parent's mark. -static __always_inline bool arming_on_fork(struct task_struct *child) +// It returns false only when the parent is armed and the child's mark failed. +// An unarmed parent (for example a runtime helper) is out of scope, not a +// failure. A new thread whose group leader is armed is covered through the +// leader (see arming_task_armed), so its own mark is best effort. +static __always_inline bool arming_on_fork(struct task_struct *child, + __u64 clone_flags) { - if (arming_task_armed(bpf_get_current_task_btf())) - return arming_mark(child); - return true; + struct task_struct *parent = bpf_get_current_task_btf(); + struct task_struct *leader; + __u32 *a; + + if (!arming_task_armed(parent)) + return true; + if (clone_flags & ARMING_CLONE_THREAD) { + leader = parent->group_leader; + a = leader ? bpf_task_storage_get(&armed_tasks, leader, 0, 0) : + NULL; + if (a && *a) { + if (!arming_fail_fork_marks) + arming_mark(child); + return true; + } + } + if (arming_fail_fork_marks) + return false; + return arming_mark(child); } #endif /* __ENFORCEMENT_ARMING_H */ diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings.go b/pkg/ebpf/gadgets/common/enforcement/bindings.go index d5bad43220..f675c3a08c 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings.go @@ -70,6 +70,18 @@ type Config struct { // ArmMap names the arming task-storage map. When set, binding a container // also arms the tasks already running in it (see ../arming.h). ArmMap string + // FaultMap names the per-slot tracking fault array (arming_faults in + // ../arming.h). When set, SlotFaults reads it. + FaultMap string +} + +// FaultMap is the part of *ebpf.Map that SlotFaults reads. +type FaultMap interface { + Lookup(key, valueOut interface{}) error + Type() ebpf.MapType + KeySize() uint32 + ValueSize() uint32 + MaxEntries() uint32 } type Status struct { @@ -90,6 +102,7 @@ type Bindings struct { desired map[execpolicy.Container]Binding active map[uint64]Binding members, armed Map + faults FaultMap ready, stopped bool startupComplete bool startupError error @@ -213,8 +226,19 @@ func (b *Bindings) bind(c Binding) error { if err := b.members.Put(id.CgroupID, bindingValue{Slot: c.Slot}); err != nil { return fmt.Errorf("writing container binding: %w", err) } - if !bound && b.armed != nil { + // A slot move arms again too: a task that the old slot left unarmed (an + // audit slot allows a fork whose child mark failed) must not escape the + // new slot. + if b.armed != nil { if err := b.armExisting(b.armed, id.CgroupPath); err != nil { + if bound && !errors.Is(err, ErrContainerGone) { + // Undo the move, so a retry moves and arms again. + if perr := b.members.Put(id.CgroupID, bindingValue{Slot: previous.Slot}); perr != nil { + b.active[id.CgroupID] = c + return errors.Join(err, perr) + } + return err + } // A container that is gone needs no protection; a startup // replay skips it (see start). if derr := b.members.Delete(id.CgroupID); derr != nil { @@ -223,6 +247,7 @@ func (b *Bindings) bind(c Binding) error { b.active[id.CgroupID] = c return errors.Join(err, derr) } + delete(b.active, id.CgroupID) return err } } @@ -245,7 +270,7 @@ func (b *Bindings) unbind(c ContainerIdentity) error { } func (b *Bindings) Start(gc operators.GadgetContext) error { - getMap := func(name string) (Map, error) { + getMap := func(name string) (*ebpf.Map, error) { value, _ := gc.GetVar(operators.MapPrefix + name) m, ok := value.(*ebpf.Map) if !ok || m == nil { @@ -261,6 +286,15 @@ func (b *Bindings) Start(gc operators.GadgetContext) error { if err != nil { return err } + var faults FaultMap + if b.config.FaultMap != "" { + if faults, err = getMap(b.config.FaultMap); err != nil { + return err + } + if faults.Type() != ebpf.Array || faults.KeySize() != 4 || faults.ValueSize() != 8 || faults.MaxEntries() < b.config.Slots { + return fmt.Errorf("invalid fault map layout") + } + } var armed Map if b.config.ArmMap != "" { if armed, err = getMap(b.config.ArmMap); err != nil { @@ -270,11 +304,27 @@ func (b *Bindings) Start(gc operators.GadgetContext) error { return fmt.Errorf("invalid arming map layout") } } - return b.start(policy, members, armed) + return b.start(policy, members, armed, faults) +} + +// SlotFaults returns the tracking faults counted for a slot since the instance +// started (see ../arming.h). A nonzero count means a task of an enforcing slot +// may be outside the rules, or was refused a fork. +func (b *Bindings) SlotFaults(slot uint32) (uint64, error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.faults == nil { + return 0, fmt.Errorf("no fault map configured") + } + var count uint64 + if err := b.faults.Lookup(slot, &count); err != nil { + return 0, fmt.Errorf("reading tracking faults: %w", err) + } + return count, nil } // start writes and freezes the policy before any binding is written. -func (b *Bindings) start(policy, members, armed Map) error { +func (b *Bindings) start(policy, members, armed Map, faults FaultMap) error { b.mu.Lock() defer b.mu.Unlock() if b.stopped { @@ -292,7 +342,7 @@ func (b *Bindings) start(policy, members, armed Map) error { if err := policy.Freeze(); err != nil { return fmt.Errorf("freezing policy: %w", err) } - b.members, b.armed = members, armed + b.members, b.armed, b.faults = members, armed, faults for _, c := range b.desired { if err := b.bind(c); err != nil && !errors.Is(err, ErrContainerGone) { return b.recordError(err) diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go index 29df707e10..43f3e8d8e6 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go @@ -67,7 +67,7 @@ func TestFreezeBeforeBindAndLifecycle(t *testing.T) { armed := &fakeMap{typ: ebpf.TaskStorage} var armedPaths []string b.armExisting = func(_ Map, path string) error { armedPaths = append(armedPaths, path); return nil } - if err := b.start(policy, members, armed); err != nil { + if err := b.start(policy, members, armed, nil); err != nil { t.Fatal(err) } if err := b.Ready(nil); err != nil { @@ -86,10 +86,17 @@ func TestFreezeBeforeBindAndLifecycle(t *testing.T) { if err := b.Bind(Binding{Container: identity}); err != nil { t.Fatal(err) } - // Moving to another slot is one map write and no second arming. - if err := b.Bind(Binding{Container: identity, Slot: 1}); err != nil || members.data[uint64(42)] != (bindingValue{Slot: 1}) || len(armedPaths) != 1 { + // Moving to another slot is one map write, and it arms again: the old + // slot may have left a task unarmed. + if err := b.Bind(Binding{Container: identity, Slot: 1}); err != nil || members.data[uint64(42)] != (bindingValue{Slot: 1}) || len(armedPaths) != 2 { t.Fatalf("slot move: %v %v armed=%v", err, members.data, armedPaths) } + // A move whose arming fails is undone, so a retry moves and arms again. + b.armExisting = func(Map, string) error { return errors.New("walk failed") } + if err := b.Bind(Binding{Container: identity, Slot: 0}); err == nil || members.data[uint64(42)] != (bindingValue{Slot: 1}) { + t.Fatalf("failed move: %v %v", err, members.data) + } + b.armExisting = func(_ Map, path string) error { armedPaths = append(armedPaths, path); return nil } if err := b.Bind(Binding{Container: identity, Slot: 2}); err == nil { t.Fatal("slot out of range must fail") } @@ -109,7 +116,7 @@ func TestFreezeBeforeBindAndLifecycle(t *testing.T) { func TestFailedArmingRemovesBinding(t *testing.T) { b, policy, members, _ := newFixture(t, func(Map) error { return nil }) b.armExisting = func(Map, string) error { return errors.New("no cgroup") } - if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}); err == nil { + if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}, nil); err == nil { t.Fatal("arming failure must fail startup") } if len(members.data) != 0 { @@ -120,11 +127,11 @@ func TestFailedArmingRemovesBinding(t *testing.T) { func TestRejectsWrongBindingLayoutAndPolicyErrors(t *testing.T) { b, policy, members, log := newFixture(t, func(Map) error { return nil }) members.max = 8 - if err := b.start(policy, members, nil); err == nil || len(*log) != 0 { + if err := b.start(policy, members, nil, nil); err == nil || len(*log) != 0 { t.Fatalf("wrong binding layout must fail before writes: %v %v", err, *log) } b, policy, members, log = newFixture(t, func(Map) error { return errors.New("bad layout") }) - if err := b.start(policy, members, nil); err == nil || len(*log) != 0 { + if err := b.start(policy, members, nil, nil); err == nil || len(*log) != 0 { t.Fatalf("policy writer error must stop startup: %v %v", err, *log) } } @@ -138,7 +145,7 @@ func TestNewValidates(t *testing.T) { func TestStartSkipsGoneContainer(t *testing.T) { b, policy, members, _ := newFixture(t, func(Map) error { return nil }) b.armExisting = func(Map, string) error { return ErrContainerGone } - if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}); err != nil { + if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}, nil); err != nil { t.Fatalf("a gone container must not fail startup: %v", err) } if err := b.Ready(nil); err != nil || len(members.data) != 0 { @@ -152,3 +159,30 @@ func TestStartSkipsGoneContainer(t *testing.T) { t.Fatalf("status %+v", st) } } + +type fakeFaults map[uint32]uint64 + +func (f fakeFaults) Lookup(k, v interface{}) error { + *v.(*uint64) = f[k.(uint32)] + return nil +} +func (fakeFaults) Type() ebpf.MapType { return ebpf.Array } +func (fakeFaults) KeySize() uint32 { return 4 } +func (fakeFaults) ValueSize() uint32 { return 8 } +func (fakeFaults) MaxEntries() uint32 { return 64 } + +func TestSlotFaults(t *testing.T) { + b, policy, members, _ := newFixture(t, func(Map) error { return nil }) + if _, err := b.SlotFaults(0); err == nil { + t.Fatal("no fault map must be an error, not zero faults") + } + if err := b.start(policy, members, nil, fakeFaults{1: 3}); err != nil { + t.Fatal(err) + } + if n, err := b.SlotFaults(0); err != nil || n != 0 { + t.Fatalf("slot 0: %d %v", n, err) + } + if n, err := b.SlotFaults(1); err != nil || n != 3 { + t.Fatalf("slot 1: %d %v", n, err) + } +} diff --git a/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml b/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml index 11eb6602bb..659bb5270b 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml +++ b/pkg/ebpf/gadgets/enforce_privilege/gadget.yaml @@ -22,3 +22,7 @@ params: key: host-mntns-id defaultValue: "0" description: mount namespace inode of the container runtime; 0 uses the global init task's + arming_fail_fork_marks: + key: arming-fail-fork-marks + defaultValue: "false" + description: test only - fail every child arming mark at fork (fault injection) diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go index 60db0f2e7e..a292a917a4 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go @@ -309,11 +309,16 @@ func compileSlot(slot uint32, docs []Document, generation uint64, c *Compiled, o } } - entries := []KernelEntry{meta} keys := make([]KernelKey, 0, len(folded)) - for k := range folded { + for k, v := range folded { keys = append(keys, k) + // The META mode tells the program whether the slot enforces at + // all: an audit policy can carry an enforce rule. + if v.Rank == RankDenyEnforce { + meta.Value.Mode = KernelModeEnforce + } } + entries := []KernelEntry{meta} sort.Slice(keys, func(i, j int) bool { a, b := keys[i], keys[j] if a.Kind != b.Kind { diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go index d153fd10db..1511773c85 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go @@ -69,7 +69,8 @@ func TestCompileFoldsMostLimiting(t *testing.T) { t.Fatal(err) } meta, _ := lookup(c, KernelKey{Kind: KindMeta}) - if meta.Generation != 7 || meta.Mode != KernelModeAudit { + // The policy audits, but one rule enforces: the slot enforces. + if meta.Generation != 7 || meta.Mode != KernelModeEnforce { t.Fatalf("meta %+v", meta) } // Deny+enforce beats the allow on the same key, independent of order. diff --git a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c index 13d0ae97a4..c7f894dc3c 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c +++ b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c @@ -31,13 +31,16 @@ char LICENSE[] SEC("license") = "GPL"; GADGET_PARAM(host_mntns_id); +GADGET_PARAM(arming_fail_fork_marks); #define EPERM 1 +#define ENOMEM 12 #define CAP_SYS_ADMIN 21 #define CAP_OPT_NOAUDIT (1UL << 1) #define PTRACE_MODE_ATTACH 0x02 #define CLONE_NEWTIME 0x00000080ULL +#define CLONE_INTO_CGROUP 0x200000000ULL #define CLONE_NEWNS 0x00020000ULL #define CLONE_NEWCGROUP 0x02000000ULL #define CLONE_NEWUTS 0x04000000ULL @@ -461,11 +464,48 @@ static __always_inline int check_syscall(void *ctx, enum priv_hook hook, /* Arming */ +// priv_enforcing_slot returns true, with the slot, when the current task is +// in a bound cgroup whose slot enforces. +static __always_inline bool priv_enforcing_slot(__u32 *slot) +{ + struct policy_value *header; + __u64 cgid = 0; + + header = priv_header(&cgid, slot); + return header && header->mode == POLICY_MODE_ENFORCE; +} + +// The exec hook cannot fail the exec. A lost mark in an enforcing slot is a +// tracking fault: the owner reports the binding as not confirmed. SEC("lsm/bprm_committed_creds") int BPF_PROG(ig_priv_exec) { - if (!arming_on_exec()) + __u32 slot = 0; + + if (!arming_on_exec()) { verdict_count(VERDICT_COUNT_SECTION_ERROR); + if (priv_enforcing_slot(&slot)) + arming_slot_fault(slot); + } + return 0; +} + +// priv_lost_child handles an armed parent whose child could not be marked. +// An unmarked child would escape every rule, so an enforcing slot refuses the +// fork and counts a tracking fault. An audit slot or an unbound cgroup allows +// it; the failure is in the section error counter only. A clone into another +// cgroup is refused too, because the child's binding is not known here. +static __always_inline int priv_lost_child(unsigned long clone_flags) +{ + __u32 slot = 0; + + verdict_count(VERDICT_COUNT_SECTION_ERROR); + if (priv_enforcing_slot(&slot)) { + arming_slot_fault(slot); + return -ENOMEM; + } + if ((__u64)clone_flags & CLONE_INTO_CGROUP) + return -ENOMEM; return 0; } @@ -511,8 +551,8 @@ static __always_inline int do_task_alloc(void *ctx, struct task_struct *task, enum priv_syscall sc; __u64 flags; - if (!arming_on_fork(task)) - verdict_count(VERDICT_COUNT_SECTION_ERROR); + if (!arming_on_fork(task, clone_flags) && priv_lost_child(clone_flags)) + return -ENOMEM; // userns_create judges CLONE_NEWUSER. flags = clone_flags & CLONE_NS_MASK & ~CLONE_NEWUSER; diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go index df8501e046..c38d7d732e 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime.go @@ -126,6 +126,7 @@ func (*operator) InstantiateDataOperator(gc operators.GadgetContext, _ api.Param PolicyMap: "policy", WritePolicy: writePolicy(config.Policy.Entries), ArmMap: "armed_tasks", + FaultMap: "arming_faults", }) if err != nil { return nil, err diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go index 4dede884d2..c35adfb140 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go @@ -15,6 +15,9 @@ package runtime // PRIVILEGE_IMAGE image name in the archive // PRIVILEGE_CONTAINER_ID full Docker container ID // PRIVILEGE_CGROUP_PATH its cgroup v2 path under /sys/fs/cgroup, with a leading "/" (as in /proc//cgroup) +// +// TestForkMarkFaultInjection uses the same fixture. The container needs a +// POSIX sh. import ( "context" @@ -43,34 +46,34 @@ import ( privpolicy "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/policy" ) -func TestGadgetWithInspektorGadgetAlone(t *testing.T) { +// kernelFixture runs one gadget instance on the fixture container. +type kernelFixture struct { + ctx context.Context + id string + container ContainerIdentity + bindings *enforcement.Bindings + verdicts atomic.Int32 +} + +func newKernelFixture(t *testing.T) *kernelFixture { + t.Helper() if os.Getenv("PRIVILEGE_KERNEL_TEST") != "1" { t.Skip("explicit disposable kernel fixture opt-in required") } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - - f, err := os.Open("../examples/policy.yaml") - if err != nil { - t.Fatal(err) - } - doc, err := privpolicy.Parse(f) - f.Close() - if err != nil { - t.Fatal(err) - } - compiled, err := privpolicy.CompileKernel(doc, 1) - if err != nil { - t.Fatal(err) - } + t.Cleanup(cancel) id, cgroupPath := os.Getenv("PRIVILEGE_CONTAINER_ID"), os.Getenv("PRIVILEGE_CGROUP_PATH") var st unix.Stat_t if err := unix.Stat(filepath.Join("/sys/fs/cgroup", cgroupPath), &st); err != nil { t.Fatal(err) } - container := ContainerIdentity{Runtime: "docker", ID: id, CgroupID: st.Ino, CgroupPath: cgroupPath} + return &kernelFixture{ctx: ctx, id: id, container: ContainerIdentity{Runtime: "docker", ID: id, CgroupID: st.Ino, CgroupPath: cgroupPath}} +} - store, err := orasoci.NewFromTar(ctx, os.Getenv("PRIVILEGE_IMAGE_ARCHIVE")) +// start runs the gadget with the compiled documents and extra ebpf params. +func (f *kernelFixture) start(t *testing.T, name string, compiled privpolicy.Compiled, extra map[string]string) { + t.Helper() + store, err := orasoci.NewFromTar(f.ctx, os.Getenv("PRIVILEGE_IMAGE_ARCHIVE")) if err != nil { t.Fatal(err) } @@ -78,7 +81,7 @@ func TestGadgetWithInspektorGadgetAlone(t *testing.T) { if err := rt.Init(nil); err != nil { t.Fatal(err) } - defer rt.Close() + t.Cleanup(func() { rt.Close() }) manager, err := instancemanager.New(rt) if err != nil { t.Fatal(err) @@ -91,15 +94,13 @@ func TestGadgetWithInspektorGadgetAlone(t *testing.T) { if err := handler.Init(params); err != nil { t.Fatal(err) } - var verdicts atomic.Int32 - var bindings *enforcement.Bindings events := simple.New("privilege-test-verdicts", simple.WithPriority(100), simple.OnInit(func(gc operators.GadgetContext) error { var err error - if bindings, err = GetBindings(gc); err != nil { + if f.bindings, err = GetBindings(gc); err != nil { return err } return gc.GetDataSources()["verdict"].Subscribe(func(datasource.DataSource, datasource.Data) error { - verdicts.Add(1) + f.verdicts.Add(1) return nil }, 100) })) @@ -107,33 +108,104 @@ func TestGadgetWithInspektorGadgetAlone(t *testing.T) { if err != nil { t.Fatal(err) } - desired := &api.GadgetInstance{Id: "privilege-kernel-test", GadgetConfig: &api.GadgetRunRequest{Version: api.VersionGadgetRunProtocol, ImageName: os.Getenv("PRIVILEGE_IMAGE"), - ParamValues: map[string]string{"operator.oci.ebpf.host-mntns-id": strconv.FormatUint(runtimeMntns, 10)}}} - if err := manager.StartGadget(ctx, desired, gadgetcontext.WithOrasReadonlyTarget(store), + values := map[string]string{"operator.oci.ebpf.host-mntns-id": strconv.FormatUint(runtimeMntns, 10)} + for k, v := range extra { + values["operator.oci.ebpf."+k] = v + } + desired := &api.GadgetInstance{Id: name, GadgetConfig: &api.GadgetRunRequest{Version: api.VersionGadgetRunProtocol, ImageName: os.Getenv("PRIVILEGE_IMAGE"), ParamValues: values}} + if err := manager.StartGadget(f.ctx, desired, gadgetcontext.WithOrasReadonlyTarget(store), gadgetcontext.WithAdditionalDataOperators(handler, New(), events), WithConfig(Config{Policy: compiled})); err != nil { t.Fatal(err) } - defer manager.StopGadget(context.Background(), desired.Id) + t.Cleanup(func() { manager.StopGadget(context.Background(), desired.Id) }) +} - unshare := func() error { return exec.CommandContext(ctx, "docker", "exec", id, "unshare", "-m", "true").Run() } +func (f *kernelFixture) exec(args ...string) error { + return exec.CommandContext(f.ctx, "docker", append([]string{"exec", f.id}, args...)...).Run() +} + +func TestGadgetWithInspektorGadgetAlone(t *testing.T) { + f := newKernelFixture(t) + file, err := os.Open("../examples/policy.yaml") + if err != nil { + t.Fatal(err) + } + doc, err := privpolicy.Parse(file) + file.Close() + if err != nil { + t.Fatal(err) + } + compiled, err := privpolicy.CompileKernel(doc, 1) + if err != nil { + t.Fatal(err) + } + f.start(t, "privilege-kernel-test", compiled, nil) + + unshare := func() error { return f.exec("unshare", "-m", "true") } if err := unshare(); err != nil { t.Fatalf("unbound container must not be judged: %v", err) } - if err := bindings.Bind(enforcement.Binding{Container: container}); err != nil { + if err := f.bindings.Bind(enforcement.Binding{Container: f.container}); err != nil { t.Fatal(err) } if unshare() == nil { t.Fatal("the example policy denies unshare(CLONE_NEWNS) in a bound container") } - if err := bindings.Unbind(container); err != nil { + if err := f.bindings.Unbind(f.container); err != nil { t.Fatal(err) } if err := unshare(); err != nil { t.Fatalf("unbound again must be allowed: %v", err) } time.Sleep(500 * time.Millisecond) - if verdicts.Load() == 0 { + if f.verdicts.Load() == 0 { t.Fatal("no verdict event for the deny") } } + +// TestForkMarkFaultInjection makes every child arming mark fail at fork. An +// armed shell that forks must then be refused in enforce mode, with a tracking +// fault for the slot, and allowed in audit mode, with no tracking fault. +func TestForkMarkFaultInjection(t *testing.T) { + for _, mode := range []string{"enforce", "audit"} { + t.Run(mode, func(t *testing.T) { + f := newKernelFixture(t) + doc := privpolicy.Document{Version: 1, Name: "fault-" + mode, Policy: privpolicy.Policy{Mode: mode, Privilege: privpolicy.Section{Default: "allow"}}} + compiled, err := privpolicy.CompileKernel(doc, 1) + if err != nil { + t.Fatal(err) + } + f.start(t, "privilege-fault-"+mode, compiled, map[string]string{"arming-fail-fork-marks": "true"}) + fork := func() error { return f.exec("sh", "-c", "true & wait") } + if err := fork(); err != nil { + t.Fatalf("an unbound container must not be judged: %v", err) + } + if err := f.bindings.Bind(enforcement.Binding{Container: f.container}); err != nil { + t.Fatal(err) + } + defer f.bindings.Unbind(f.container) + err = fork() + faults, ferr := f.bindings.SlotFaults(0) + if ferr != nil { + t.Fatal(ferr) + } + switch mode { + case "enforce": + if err == nil { + t.Fatal("a fork whose child cannot be armed must be refused") + } + if faults == 0 { + t.Fatal("the refused fork must count a tracking fault") + } + case "audit": + if err != nil { + t.Fatalf("audit mode must allow the fork: %v", err) + } + if faults != 0 { + t.Fatalf("audit faults must not block: %d", faults) + } + } + }) + } +} From 164ac3312c2c4cd21738e47facb425a8e63ccfcd Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:19:44 +0200 Subject: [PATCH 20/54] test(merge): shared merge contract and test vectors One section-neutral vector file (policy combination to expected verdict) and a CONTRACT doc that states the merge rule and where it differs from the CRD text. The privilege compiler tests run the vectors. No change to the semantics. The README documents the lost-arming handling, the child cgroup scope and the follow-ups. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/common/README.md | 5 +- pkg/ebpf/gadgets/common/merge/CONTRACT.md | 35 +++++ pkg/ebpf/gadgets/common/merge/vectors.yaml | 107 +++++++++++++++ pkg/ebpf/gadgets/enforce_privilege/README.md | 42 ++++-- .../enforce_privilege/policy/contract_test.go | 126 ++++++++++++++++++ 5 files changed, 305 insertions(+), 10 deletions(-) create mode 100644 pkg/ebpf/gadgets/common/merge/CONTRACT.md create mode 100644 pkg/ebpf/gadgets/common/merge/vectors.yaml create mode 100644 pkg/ebpf/gadgets/enforce_privilege/policy/contract_test.go diff --git a/pkg/ebpf/gadgets/common/README.md b/pkg/ebpf/gadgets/common/README.md index 0359e1792f..cce63570ee 100644 --- a/pkg/ebpf/gadgets/common/README.md +++ b/pkg/ebpf/gadgets/common/README.md @@ -17,8 +17,9 @@ and keeps its path rules in an array that `policy.h` does not describe. | `bindings.h` | `bindings` map: cgroup id → the container's policy slot; optional parent-cgroup depth | | `policy.h` | policy modes (0 off, 1 audit, 2 enforce), rank resolution (explicit deny > allow > default, one mode per rule), shared policy value | | `verdict.h` | first verdict fields (`generation`, `cgroup`, `deny`, `mode`), per-CPU `counters` (denied, submitted, dropped, allowed, section error, rate limited), 100 reports per CPU per second | -| `arming.h` | arming rule: judge only tasks that executed outside the runtime mount namespace, and their children | -| `enforcement/` | Go: the binding controller of one node-wide instance (policy written and frozen before any binding, live `Bind`/`Unbind` of a container to a slot, readiness, arming of tasks already running) | +| `arming.h` | arming rule: judge only tasks that executed outside the runtime mount namespace, and their children; `arming_faults`, tracking faults per slot | +| `merge/` | the merge contract for several policies on one container, and its shared test vectors | +| `enforcement/` | Go: the binding controller of one node-wide instance (policy written and frozen before any binding, live `Bind`/`Unbind` of a container to a slot, readiness, arming of tasks already running, `SlotFaults`) | Section-specific parts (the policy key layout, hooks, compiler) stay in each gadget. diff --git a/pkg/ebpf/gadgets/common/merge/CONTRACT.md b/pkg/ebpf/gadgets/common/merge/CONTRACT.md new file mode 100644 index 0000000000..0043693e17 --- /dev/null +++ b/pkg/ebpf/gadgets/common/merge/CONTRACT.md @@ -0,0 +1,35 @@ +# Merge contract + +Several policies can apply to one container. Every section compiler merges them +with the same rule, and `vectors.yaml` holds the shared test vectors. A section +compiler that merges policies runs the vectors in its tests (the privilege +compiler: `enforce_privilege/policy/contract_test.go`). + +## Rule + +1. Rules are the union of every policy's rules. Each rule keeps its own mode + (else its policy's mode). +2. On one key the most limiting rule wins: deny (enforce) > deny (audit) > + allow. +3. When no rule matches a key, the section default applies. The merged default + is deny if any policy's default is deny, else allow. Its mode is the + strictest mode of the policies whose default equals the merged default. +4. An explicit allow beats another policy's default deny. +5. An audit deny never weakens another policy's enforce default deny. When a + key is denied by rules only in audit mode, the enforce default deny of each + policy without a rule on that key still applies. So adding an audit policy + never makes the merged result less strict. +6. Qualified keys (privilege: namespace flags) are judged per qualifier, with + rules 1–5 for each qualifier. + +## Where it differs from the CRD text + +The CRD design says the section defaults apply only when no rule matches. Read +literally, a matching audit deny of one policy then hides another policy's +enforce default deny, and the merged result is audit only. Rule 5 differs: it +keeps the enforce default deny. The vector "an audit deny never weakens another +policy's enforce default deny" covers it. + +Rule 5 is pending a product decision. Until then, a change to it changes +`vectors.yaml` and this file in the same commit, so the sections cannot diverge +silently. diff --git a/pkg/ebpf/gadgets/common/merge/vectors.yaml b/pkg/ebpf/gadgets/common/merge/vectors.yaml new file mode 100644 index 0000000000..37b1421002 --- /dev/null +++ b/pkg/ebpf/gadgets/common/merge/vectors.yaml @@ -0,0 +1,107 @@ +# Merge contract vectors (see CONTRACT.md): a combination of policies on one +# container and the verdict expected for each call. Every section compiler that +# merges policies runs these vectors in its tests. +# +# Section-neutral names: +# key a rule target (privilege: a syscall; k1 = mount, k2 = unshare) +# qual a qualifier that narrows a key (privilege: a namespace flag; +# q1 = CLONE_NEWNS, q2 = CLONE_NEWNET). A rule without qual +# matches every qualifier. A call with quals is judged per +# qualifier and the most limiting result wins. +# verdict allow | deny-audit | deny-enforce +# A rule without mode takes its policy's mode. +vectors: + - name: one policy, enforce default deny + policies: + - {mode: enforce, default: deny} + checks: + - {key: k1, verdict: deny-enforce} + + - name: one policy, its own audit rule beats its default + policies: + - mode: enforce + default: deny + rules: + - {key: k1, action: deny, mode: audit} + checks: + - {key: k1, verdict: deny-audit} + + - name: an audit deny never weakens another policy's enforce default deny + policies: + - mode: audit + default: allow + rules: + - {key: k1, action: deny} + - {mode: enforce, default: deny} + checks: + - {key: k1, verdict: deny-enforce} + + - name: two audit policies stay audit + policies: + - mode: audit + default: allow + rules: + - {key: k1, action: deny} + - {mode: audit, default: deny} + checks: + - {key: k1, verdict: deny-audit} + + - name: an explicit allow beats another policy's default deny + policies: + - mode: enforce + default: allow + rules: + - {key: k1, action: allow} + - {mode: enforce, default: deny} + checks: + - {key: k1, verdict: allow} + - {key: k2, quals: [q1], verdict: deny-enforce} + + - name: an explicit deny beats another policy's allow + policies: + - mode: enforce + default: allow + rules: + - {key: k1, action: deny} + - mode: enforce + default: allow + rules: + - {key: k1, action: allow} + checks: + - {key: k1, verdict: deny-enforce} + + - name: the merged default takes the mode of the policies that agree + policies: + - {mode: audit, default: deny} + - {mode: enforce, default: allow} + checks: + - {key: k1, verdict: deny-audit} + + - name: per qualifier, an audit deny on one qualifier under an allow of the key + policies: + - mode: audit + default: allow + rules: + - {key: k2, qual: q1, action: deny} + - mode: enforce + default: deny + rules: + - {key: k2, action: allow} + checks: + - {key: k2, quals: [q1], verdict: deny-audit} + - {key: k2, quals: [q2], verdict: allow} + + - name: per qualifier, an audit deny of the key under an allow of one qualifier + policies: + - mode: audit + default: allow + rules: + - {key: k2, action: deny} + - mode: enforce + default: deny + rules: + - {key: k2, qual: q1, action: allow} + checks: + - {key: k2, quals: [q1], verdict: deny-audit} + - {key: k2, quals: [q2], verdict: deny-enforce} + - {key: k2, quals: [q1, q2], verdict: deny-enforce} diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index 920bdb2658..5f4dcd479c 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -16,7 +16,9 @@ node: - The policy map is frozen per instance. A combination the running instance does not have (a new or edited policy) needs a new instance: the owner starts one with every slot in use and every container bound, and then stops the old - one, so both enforce during the overlap. + one, so both enforce during the overlap. This whole-instance replacement is a + staging choice: the owner hides it behind its backend interface, so a later + in-place policy update can replace it. - An owner can create a slot before a container starts, so the container binds with a live write. @@ -73,7 +75,9 @@ policy: own name, origin and mode); a default is deny if any document denies, with the strictest mode among the documents that agree; a monitoring (audit) deny rule never weakens another document's enforce deny, including its enforce default - deny; an explicit allow still beats another document's default deny. + deny; an explicit allow still beats another document's default deny. The + rule and its test vectors are shared with other sections: + [merge contract](../common/merge/CONTRACT.md). ## How it decides @@ -86,7 +90,23 @@ policy: `host-mntns-id` gadget parameter; the owner sets it to the mount namespace of PID 1 as it sees it (in kind, the node container's). When it is 0, the program uses the global init task's. -- **Scope.** A bound cgroup also covers up to 31 levels of child cgroups. +- **Lost arming.** The kernel can fail to allocate a task's mark (memory + pressure). When an armed task forks and the child's mark fails, an enforcing + slot (one with any enforce entry) refuses the fork (`-ENOMEM`): an unmarked + child would escape every rule. A clone into another cgroup + (`CLONE_INTO_CGROUP`) is refused the same way. An audit-only slot, or an + unbound cgroup, allows the fork. A new thread of an armed group leader is + covered through the leader, so it is never refused. An unarmed parent (for + example a runtime helper) is out of scope, not a failure. Every failed mark is + counted in `counters` slot 4. Moving a container to another slot arms its + running tasks again. +- **Tracking faults.** A failed mark in an enforcing slot (a refused fork, or a + lost mark at exec, which cannot be refused) is also counted per slot in + `arming_faults`. The owner reads it with `Bindings.SlotFaults`: a nonzero + count means the slot's protection is not confirmed. A new instance starts at + zero and arms the running tasks again. Audit-only faults do not count there. +- **Scope.** A bound cgroup also covers up to 31 levels of child cgroups. This is + by design and differs from enforce_exec, which binds the exact cgroup only. - **Capabilities.** A `capable()` call is judged only when the task uses its own credentials (overlayfs copy-up uses the mounter's) and the call is not a no-audit probe. @@ -99,10 +119,16 @@ policy: ## Known limits -- **Failed arming mark.** If the kernel cannot allocate task storage (memory - pressure), the task stays unarmed and is not judged until its next exec. The - failure is counted in `counters` slot 4. A later check cannot tell such a - task apart from the runtime's own setup, so it is not retried. +- **Lost mark at exec.** The exec hook cannot fail the exec, so a task whose + mark fails there stays unarmed until its next exec. In an enforcing slot this + is a tracking fault (see above). A later check cannot tell such a task apart + from the runtime's own setup, so the gadget does not retry it; a new instance + arms it again. +- **Fault injection.** The `arming-fail-fork-marks` parameter makes every child + mark fail at fork. It exists for tests (`runtime/runtime_kernel_test.go`); + owners do not set it. +- **Not in this gadget yet:** export of verdicts as events (the owner logs + them), and kernels older than 6.1. - **No-audit capability checks** (`CAP_OPT_NOAUDIT`) are excluded from capability enforcement. Most are kernel probes (for example the overcommit check), and judging them would deny normal work in every bound container. Some @@ -127,7 +153,7 @@ policy: | Map | Content | |---|---| | `policy` | frozen hash: `{slot, kind, id, qual, name}` → `{rank, rule_id, origin_id}`; each slot's META entry holds generation and mode (`privilege_policy.h`) | -| `bindings`, `armed_tasks`, `counters` | from the common library | +| `bindings`, `armed_tasks`, `arming_faults`, `counters` | from the common library | The `verdict` datasource shares `generation`, `cgroup`, `deny`, `mode` and `host_pid` with enforce_exec and adds `rule_id`, `origin_id`, `is_default`, diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/contract_test.go b/pkg/ebpf/gadgets/enforce_privilege/policy/contract_test.go new file mode 100644 index 0000000000..7a4a52b67e --- /dev/null +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/contract_test.go @@ -0,0 +1,126 @@ +// Copyright 2026 The Kubescape Authors +// SPDX-License-Identifier: Apache-2.0 + +package privpolicy + +import ( + "bytes" + "os" + "testing" + + "gopkg.in/yaml.v3" +) + +// The shared merge contract vectors (../../common/merge/CONTRACT.md), mapped to +// privilege targets. +var ( + contractKeys = map[string]string{"k1": "mount", "k2": "unshare"} + contractQuals = map[string]string{"q1": "CLONE_NEWNS", "q2": "CLONE_NEWNET"} + contractRanks = map[string]uint8{"allow": RankAllow, "deny-audit": RankDenyAudit, "deny-enforce": RankDenyEnforce} +) + +type contractVectors struct { + Vectors []struct { + Name string `yaml:"name"` + Policies []struct { + Mode string `yaml:"mode"` + Default string `yaml:"default"` + Rules []struct { + Key string `yaml:"key"` + Qual string `yaml:"qual"` + Action string `yaml:"action"` + Mode string `yaml:"mode"` + } `yaml:"rules"` + } `yaml:"policies"` + Checks []struct { + Key string `yaml:"key"` + Quals []string `yaml:"quals"` + Verdict string `yaml:"verdict"` + } `yaml:"checks"` + } `yaml:"vectors"` +} + +// contractRank resolves a call like the program: per qualifier, the rules on +// the bare key and on the qualified key, else the default; the most limiting +// qualifier wins. +func contractRank(c Compiled, syscall string, flags []string) uint8 { + def, _ := lookup(c, KernelKey{Kind: KindDefault, ID: KindSyscall}) + bare, _ := lookup(c, KernelKey{Kind: KindSyscall, ID: Syscalls[syscall]}) + if len(flags) == 0 { + if bare.Rank != RankNone { + return bare.Rank + } + return def.Rank + } + var result uint8 + for _, f := range flags { + q, _ := lookup(c, KernelKey{Kind: KindSyscall, ID: Syscalls[syscall], Qual: NamespaceFlags[f]}) + r := max(bare.Rank, q.Rank) + if r == RankNone { + r = def.Rank + } + result = max(result, r) + } + return result +} + +func TestMergeContractVectors(t *testing.T) { + data, err := os.ReadFile("../../common/merge/vectors.yaml") + if err != nil { + t.Fatal(err) + } + var file contractVectors + decoder := yaml.NewDecoder(bytes.NewReader(data)) + decoder.KnownFields(true) + if err := decoder.Decode(&file); err != nil { + t.Fatal(err) + } + key := func(k string) string { + if contractKeys[k] == "" { + t.Fatalf("unknown key %q", k) + } + return contractKeys[k] + } + qual := func(q string) string { + if contractQuals[q] == "" { + t.Fatalf("unknown qual %q", q) + } + return contractQuals[q] + } + if len(file.Vectors) == 0 { + t.Fatal("no vectors") + } + for _, v := range file.Vectors { + t.Run(v.Name, func(t *testing.T) { + var docs []Document + for i, p := range v.Policies { + d := Document{Version: 1, Name: string(rune('a' + i)), Policy: Policy{Mode: p.Mode, Privilege: Section{Default: p.Default}}} + for j, r := range p.Rules { + rule := Rule{Name: "r" + string(rune('0'+j)), Action: r.Action, Mode: r.Mode, Target: Target{Syscall: []string{key(r.Key)}}} + if r.Qual != "" { + rule.Target.Flags = []string{qual(r.Qual)} + } + d.Policy.Privilege.Rules = append(d.Policy.Privilege.Rules, rule) + } + docs = append(docs, d) + } + c, err := CompileSlots([][]Document{docs}, 1) + if err != nil { + t.Fatal(err) + } + for _, check := range v.Checks { + var flags []string + for _, q := range check.Quals { + flags = append(flags, qual(q)) + } + want, ok := contractRanks[check.Verdict] + if !ok { + t.Fatalf("unknown verdict %q", check.Verdict) + } + if got := contractRank(c, key(check.Key), flags); got != want { + t.Errorf("%s %v: rank %d, want %s", check.Key, check.Quals, got, check.Verdict) + } + } + }) + } +} From a2c1d9d203942dc2cd62eaab587546e8548cd7fc Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:19:44 +0200 Subject: [PATCH 21/54] fix(sandboxpolicy): a tracking fault leaves privilege protection unconfirmed Replace and Attest read the slot's tracking faults. A fault makes the binding ProtectionUnconfirmed; the next same-plan Replace starts a new instance, which arms the running tasks again. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/igbackend/privilege.go | 18 ++++++++++-- .../igbackend/privilege_native.go | 11 ++++++++ pkg/sandboxpolicy/igbackend/privilege_test.go | 28 +++++++++++++++++++ pkg/sandboxpolicy/privilege.go | 5 ++++ 4 files changed, 60 insertions(+), 2 deletions(-) diff --git a/pkg/sandboxpolicy/igbackend/privilege.go b/pkg/sandboxpolicy/igbackend/privilege.go index ce1088a970..3ddcf692c7 100644 --- a/pkg/sandboxpolicy/igbackend/privilege.go +++ b/pkg/sandboxpolicy/igbackend/privilege.go @@ -26,6 +26,9 @@ type PrivilegeEngine interface { type PrivilegeInstance interface { Ready() error + // SlotHealth fails when the instance counted a tracking fault in the slot: + // a task of an enforcing slot may be untracked, or a fork was refused. + SlotHealth(slot uint32) error Bind(enforcement.Binding) error Unbind(enforcement.ContainerIdentity) error Stop(context.Context) error @@ -148,7 +151,7 @@ func (b *PrivilegeBackend) Replace(ctx context.Context, binding sandboxpolicy.Bi var health error if current != nil { result.ActiveRevision = current.revision - health = errors.Join(b.instance.Ready(), current.check()) + health = b.health(current) result.ProtectionUnconfirmed = health != nil } if b.closed { @@ -242,7 +245,18 @@ func (b *PrivilegeBackend) Attest(_ context.Context, binding sandboxpolicy.Bindi if current == nil || current.revision != revision || b.instance == nil { return fmt.Errorf("privilege revision %q is not active", revision) } - return errors.Join(b.instance.Ready(), current.check()) + return b.health(current) +} + +// health checks the instance, the slot's tracking faults and the identity +// lease of one bound container. A fault makes the next same-plan Replace start +// a new instance, which arms the running tasks again. +func (b *PrivilegeBackend) health(p *privilegeBinding) error { + slot, ok := b.slots[p.combination] + if !ok { + return errors.Join(fmt.Errorf("privilege combination has no slot"), b.instance.Ready(), p.check()) + } + return errors.Join(b.instance.Ready(), b.instance.SlotHealth(slot), p.check()) } // Prepare declares the combinations of the pods scheduled to this node that are diff --git a/pkg/sandboxpolicy/igbackend/privilege_native.go b/pkg/sandboxpolicy/igbackend/privilege_native.go index 141df05083..4ee6e33952 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_native.go +++ b/pkg/sandboxpolicy/igbackend/privilege_native.go @@ -58,6 +58,17 @@ func (i *nativePrivilegeInstance) Ready() error { return nil } +func (i *nativePrivilegeInstance) SlotHealth(slot uint32) error { + n, err := i.bindings.SlotFaults(slot) + if err != nil { + return err + } + if n > 0 { + return fmt.Errorf("%d tracking faults in privilege slot %d", n, slot) + } + return nil +} + func (i *nativePrivilegeInstance) Bind(b enforcement.Binding) error { return i.bindings.Bind(b) } func (i *nativePrivilegeInstance) Unbind(identity enforcement.ContainerIdentity) error { diff --git a/pkg/sandboxpolicy/igbackend/privilege_test.go b/pkg/sandboxpolicy/igbackend/privilege_test.go index 1a83423292..d026563edc 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_test.go +++ b/pkg/sandboxpolicy/igbackend/privilege_test.go @@ -17,9 +17,16 @@ type fakePrivilegeInstance struct { readyErr, stopErr error stops int bound map[uint64]uint32 + faults map[uint32]uint64 } func (i *fakePrivilegeInstance) Ready() error { return i.readyErr } +func (i *fakePrivilegeInstance) SlotHealth(slot uint32) error { + if i.faults[slot] > 0 { + return errors.New("tracking fault") + } + return nil +} func (i *fakePrivilegeInstance) Bind(b enforcement.Binding) error { i.bound[b.Container.CgroupID] = b.Slot return nil @@ -237,3 +244,24 @@ func (e *readyFailEngine) Start(context.Context, string, privruntime.Config) (Pr e.instance = &fakePrivilegeInstance{readyErr: errors.New("not ready"), stopErr: errors.New("busy"), bound: map[uint64]uint32{}} return e.instance, nil } + +func TestPrivilegeTrackingFaultUnconfirmsAndHeals(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + plan := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + if _, err := b.Replace(ctx, privBinding(1), plan); err != nil { + t.Fatal(err) + } + e.instances[0].faults = map[uint32]uint64{e.instances[0].bound[1]: 1} + if b.Attest(ctx, privBinding(1), "r1") == nil { + t.Fatal("a tracking fault must fail attestation") + } + // The same plan again replaces the instance, which arms the tasks again. + if r, err := b.Replace(ctx, privBinding(1), plan); err != nil || !r.Committed || len(e.instances) != 2 { + t.Fatalf("heal: %+v %v instances=%d", r, err, len(e.instances)) + } + if err := b.Attest(ctx, privBinding(1), "r1"); err != nil { + t.Fatalf("the new instance has no fault: %v", err) + } +} diff --git a/pkg/sandboxpolicy/privilege.go b/pkg/sandboxpolicy/privilege.go index 42e2e8e7a1..89f9622939 100644 --- a/pkg/sandboxpolicy/privilege.go +++ b/pkg/sandboxpolicy/privilege.go @@ -11,6 +11,11 @@ import ( // the privilege gadget's own policy document. name qualifies rule names in // events. Actor and path predicates are not supported by the gadget yet and // are rejected. +// +// Several policies on one container merge per the shared merge contract +// (pkg/ebpf/gadgets/common/merge/CONTRACT.md). It differs from the CRD design +// text (sandbox-policy-crd.md, "Multiple policies matching one pod merge"): an +// audit deny rule never weakens another policy's enforce default deny. func PrivilegeDocument(name string, mode api.PolicyMode, section api.PrivilegeSection) (privpolicy.Document, error) { doc := privpolicy.Document{Name: name, Version: 1, Policy: privpolicy.Policy{Mode: string(mode), Privilege: privpolicy.Section{Default: string(section.Default)}}} for _, r := range section.Rules { From 6605531781cda70ca17b81b92dec9d889885a02f Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:28:16 +0200 Subject: [PATCH 22/54] fix(enforcement): a same-slot retry arms again after a failed arming When arming failed and the kernel entry could not be removed (or a slot move could not be undone), the binding stayed tracked, and a retry with the same slot returned early without arming. The controller now marks such cgroups as unarmed, so the retry arms them. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- .../gadgets/common/enforcement/bindings.go | 20 +++++++++---- .../common/enforcement/bindings_test.go | 30 +++++++++++++++++++ 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings.go b/pkg/ebpf/gadgets/common/enforcement/bindings.go index f675c3a08c..e578bcaf68 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings.go @@ -97,10 +97,13 @@ type Status struct { // Bindings serializes all writes. Map handles are borrowed from the eBPF operator. type Bindings struct { - mu sync.Mutex - config Config - desired map[execpolicy.Container]Binding - active map[uint64]Binding + mu sync.Mutex + config Config + desired map[execpolicy.Container]Binding + active map[uint64]Binding + // unarmed marks bound cgroups whose running tasks the last bind did + // not arm, so a retry with the same slot arms them. + unarmed map[uint64]bool members, armed Map faults FaultMap ready, stopped bool @@ -118,7 +121,7 @@ func New(config Config) (*Bindings, error) { if len(config.InitialBindings) > MaxBindings { return nil, fmt.Errorf("too many initial bindings") } - b := &Bindings{config: config, desired: map[execpolicy.Container]Binding{}, active: map[uint64]Binding{}, armExisting: ArmExisting} + b := &Bindings{config: config, desired: map[execpolicy.Container]Binding{}, active: map[uint64]Binding{}, unarmed: map[uint64]bool{}, armExisting: ArmExisting} for _, c := range config.InitialBindings { if err := b.validate(c); err != nil { return nil, err @@ -210,7 +213,7 @@ func (b *Bindings) bind(c Binding) error { if bound && previous.Container != id { return fmt.Errorf("cgroup identity conflicts with active binding") } - if bound && previous.Slot == c.Slot { + if bound && previous.Slot == c.Slot && !b.unarmed[id.CgroupID] { return nil } if !bound { @@ -235,6 +238,7 @@ func (b *Bindings) bind(c Binding) error { // Undo the move, so a retry moves and arms again. if perr := b.members.Put(id.CgroupID, bindingValue{Slot: previous.Slot}); perr != nil { b.active[id.CgroupID] = c + b.unarmed[id.CgroupID] = true return errors.Join(err, perr) } return err @@ -245,12 +249,15 @@ func (b *Bindings) bind(c Binding) error { // The kernel entry stays: keep it tracked so Unbind can // remove it later. b.active[id.CgroupID] = c + b.unarmed[id.CgroupID] = true return errors.Join(err, derr) } delete(b.active, id.CgroupID) + delete(b.unarmed, id.CgroupID) return err } } + delete(b.unarmed, id.CgroupID) b.active[id.CgroupID] = c return nil } @@ -266,6 +273,7 @@ func (b *Bindings) unbind(c ContainerIdentity) error { return fmt.Errorf("removing container binding: %w", err) } delete(b.active, c.CgroupID) + delete(b.unarmed, c.CgroupID) return nil } diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go index 43f3e8d8e6..d9b20c6384 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/cilium/ebpf" + execpolicy "github.com/inspektor-gadget/inspektor-gadget/gadgets/enforce_exec/policy" "golang.org/x/sys/unix" ) @@ -19,6 +20,7 @@ type fakeMap struct { frozen bool log *[]string name string + deleteErr error } func (m *fakeMap) Put(k, v interface{}) error { @@ -31,6 +33,9 @@ func (m *fakeMap) Put(k, v interface{}) error { } func (m *fakeMap) Delete(k interface{}) error { + if m.deleteErr != nil { + return m.deleteErr + } if _, ok := m.data[k]; !ok { return ebpf.ErrKeyNotExist } @@ -124,6 +129,31 @@ func TestFailedArmingRemovesBinding(t *testing.T) { } } +// When arming fails and the kernel entry cannot be removed, the binding stays +// tracked. A retry with the same slot must arm again, not return early. +func TestSameSlotRetryArmsAfterFailedArming(t *testing.T) { + b, policy, members, _ := newFixture(t, func(Map) error { return nil }) + b.config.InitialBindings, b.desired = nil, map[execpolicy.Container]Binding{} + arms := 0 + b.armExisting = func(Map, string) error { arms++; return nil } + if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}, nil); err != nil { + t.Fatal(err) + } + b.armExisting = func(Map, string) error { arms++; return errors.New("walk failed") } + members.deleteErr = errors.New("busy") + if err := b.Bind(Binding{Container: identity}); err == nil { + t.Fatal("a failed arming must fail the bind") + } + b.armExisting = func(Map, string) error { arms++; return nil } + members.deleteErr = nil + if err := b.Bind(Binding{Container: identity}); err != nil || arms != 2 { + t.Fatalf("retry must arm: %v arms=%d", err, arms) + } + if err := b.Bind(Binding{Container: identity}); err != nil || arms != 2 { + t.Fatalf("an armed binding needs no new arming: %v arms=%d", err, arms) + } +} + func TestRejectsWrongBindingLayoutAndPolicyErrors(t *testing.T) { b, policy, members, log := newFixture(t, func(Map) error { return nil }) members.max = 8 From 4bc67853d76beee5b41152f83a8cbdfe76b0eba0 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:28:16 +0200 Subject: [PATCH 23/54] fix(enforce_privilege): reject clone and clone3 rules without flags The gadget judges a clone by its namespace flags, so a plain fork is never judged. A clone or clone3 rule without flags read as "deny every fork" but did not block one. Blocking every fork is not a useful policy, so the compiler now rejects such a rule with a clear error. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/enforce_privilege/README.md | 3 ++- .../gadgets/enforce_privilege/policy/kernel_test.go | 2 ++ pkg/ebpf/gadgets/enforce_privilege/policy/policy.go | 10 ++++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index 5f4dcd479c..a20dd5ebd4 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -57,7 +57,8 @@ policy: ``` - A rule targets exactly one kind: `syscall` (optionally with `flags` for - `unshare`, `setns`, `clone`, `clone3`), `capability` (short names such as + `unshare`, `setns`, `clone`, `clone3`; required for `clone` and `clone3`, + because a plain fork is never judged), `capability` (short names such as `SYS_ADMIN`) or `mountFsType`. - Resolution: an explicit deny beats an allow, an allow beats the default, independent of rule order. Each rule has its own mode. Each namespace flag of diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go index 1511773c85..c9fc7da67b 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go @@ -125,6 +125,8 @@ func TestValidationErrors(t *testing.T) { "NUL fstype": " - {name: r, target: {mountFsType: [\"pro\\0c\"]}, action: deny}\n", "duplicate name": " - {name: r, target: {syscall: [mount]}, action: deny}\n - {name: r, target: {syscall: [bpf]}, action: deny}\n", "unknown field": " - {name: r, target: {path: [/x]}, action: deny}\n", + "bare clone": " - {name: r, target: {syscall: [clone]}, action: deny}\n", + "bare clone3": " - {name: r, target: {syscall: [unshare, clone3]}, action: allow}\n", } { if _, err := parse(base + rule); err == nil { t.Errorf("%s: expected error", name) diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go b/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go index d44d2dd39c..92b018dd3a 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/policy.go @@ -133,6 +133,16 @@ func (t Target) validate() error { return fmt.Errorf("unsupported syscall %q", s) } } + if len(t.Flags) == 0 { + for _, s := range t.Syscall { + if s == "clone" || s == "clone3" { + // The gadget judges a clone by its namespace flags. A rule + // without flags would read as "deny every fork", which it + // does not do. + return fmt.Errorf("a %s rule must list namespace flags: a plain fork is never judged", s) + } + } + } if len(t.Flags) > 0 { if len(t.Syscall) == 0 { return fmt.Errorf("flags qualify a syscall target only") From 772f07ed4e530b18625cebc2468e66a4ed5864fc Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:28:16 +0200 Subject: [PATCH 24/54] test(sandboxpolicy): a privilege rule on clone needs namespace flags Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/privilege_test.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 pkg/sandboxpolicy/privilege_test.go diff --git a/pkg/sandboxpolicy/privilege_test.go b/pkg/sandboxpolicy/privilege_test.go new file mode 100644 index 0000000000..f73852f550 --- /dev/null +++ b/pkg/sandboxpolicy/privilege_test.go @@ -0,0 +1,25 @@ +package sandboxpolicy + +import ( + "strings" + "testing" + + api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" +) + +func TestPrivilegeDocumentRejectsFlaglessClone(t *testing.T) { + section := func(syscall string, flags ...string) api.PrivilegeSection { + return api.PrivilegeSection{Default: api.ActionAllow, Rules: []api.PrivilegeRule{{ + Rule: api.Rule{Name: "r", Action: api.ActionDeny}, + Target: api.PrivilegeTarget{Syscall: []string{syscall}, Flags: flags}, + }}} + } + for _, sc := range []string{"clone", "clone3"} { + if _, err := PrivilegeDocument("p", api.ModeEnforce, section(sc)); err == nil || !strings.Contains(err.Error(), "plain fork") { + t.Errorf("%s without flags must be rejected with a clear error: %v", sc, err) + } + if _, err := PrivilegeDocument("p", api.ModeEnforce, section(sc, "CLONE_NEWNS")); err != nil { + t.Errorf("%s with flags: %v", sc, err) + } + } +} From 15828d06f1bec83de2e8aba3a55cfb6af8c2f683 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:39:19 +0200 Subject: [PATCH 25/54] fix(sandboxpolicy): adding a privilege policy does not restart exec When no policy object holds both sections, the exec plan is now the plan of the other policies with the revision the controller gives that plan alone. Adding or removing a separate privilege policy keeps the exec revision, so the unchanged exec instance is not restarted. Exec-only plans are unchanged. A policy object that holds both sections still restarts exec once when only its privilege section changes (its generation is part of the exec revision); this is documented as a known limit. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/sections.go | 31 +++++++++++++++++++++++++++--- pkg/sandboxpolicy/sections_test.go | 28 +++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/pkg/sandboxpolicy/sections.go b/pkg/sandboxpolicy/sections.go index 9f1432cc1c..e64ce02da7 100644 --- a/pkg/sandboxpolicy/sections.go +++ b/pkg/sandboxpolicy/sections.go @@ -14,9 +14,13 @@ import ( // // A binding whose old and new plans have no privilege section goes straight to // the exec backend, unchanged. Otherwise the plan is split into an exec plan -// and a privilege plan, each with its own revision. A policy's generation is part -// of the exec revision, so an edit restarts exec only when exec and privilege -// share one policy object; sections in separate policies stay independent. +// and a privilege plan, each with its own revision. When no policy object holds +// both sections, the exec plan and its revision are what the controller gives +// the exec policies alone, so adding or removing a privilege policy does not +// restart exec. When one object holds both, its generation is part of the exec +// revision (as in the exec-only path): an edit of only its privilege section +// restarts exec once. This is a known limit; the generation must stay in the +// plan for status staleness. // // Commit follows the Backend contract of each section: a section that returns // Committed=true is committed, even with a cleanup error. If the second section @@ -110,6 +114,27 @@ func split(plan Plan) (sectionPlans, error) { } sort.Slice(parts, func(i, j int) bool { return parts[i].Identity.UID < parts[j].Identity.UID }) s.privilege.Revision = digest(parts) + shared := false + for _, p := range plan.Policies { + if p.Privilege != nil && p.Exec.Default != "" { + shared = true + } + } + if !shared { + // No policy object carries both sections: the exec plan is the plan + // of the other policies, with the revision the controller gives + // that plan alone. Adding or removing a privilege policy then keeps + // the exec revision, so exec does not restart. + for _, p := range plan.Policies { + if p.Privilege == nil { + s.exec.Policies = append(s.exec.Policies, p) + } + } + if len(s.exec.Policies) > 0 { + s.exec.Revision = digest(s.exec.Policies) + } + return s, nil + } for _, p := range plan.Policies { if p.Exec.Default == "" { continue diff --git a/pkg/sandboxpolicy/sections_test.go b/pkg/sandboxpolicy/sections_test.go index 7b000f1d2c..86b8ad897e 100644 --- a/pkg/sandboxpolicy/sections_test.go +++ b/pkg/sandboxpolicy/sections_test.go @@ -272,3 +272,31 @@ func TestPrivilegeUnavailableIsReportedAndRefused(t *testing.T) { t.Fatalf("privilege binding must be refused: %v", err) } } + +// The controller gives a plan the digest of its policies as revision. Adding a +// privilege policy to an exec-only binding, and removing it again, must keep +// the exec revision, so exec does not restart. +func TestSectionBackendSeparatePrivilegePolicyKeepsExecRevision(t *testing.T) { + exec, privilege := &fakeBackend{plans: map[string]Plan{}}, &fakeBackend{plans: map[string]Plan{}} + s, _ := NewSectionBackend(exec, privilege, nil) + b := Binding{ContainerID: "c"} + execPolicy := CompiledPolicy{Identity: PolicyIdentity{UID: "e", Generation: 3}, Revision: "e", Exec: api.ExecSection{Default: api.ActionDeny}} + privPolicy := CompiledPolicy{Identity: PolicyIdentity{UID: "p", Generation: 1}, Revision: "p", Privilege: privilegeSection(api.ActionAllow)} + plan := func(policies ...CompiledPolicy) Plan { return Plan{Revision: digest(policies), Policies: policies} } + steps := []Plan{plan(execPolicy), plan(execPolicy, privPolicy), plan(execPolicy)} + privPolicy.Identity.Generation = 2 // an edit of the privilege policy object + privPolicy.Privilege = privilegeSection(api.ActionDeny) + steps = append(steps[:2], plan(execPolicy, privPolicy), steps[2]) + for i, p := range steps { + r, err := s.Replace(context.Background(), b, p) + if err != nil || !r.Committed || r.ActiveRevision != p.Revision { + t.Fatalf("step %d: %+v %v", i, r, err) + } + } + if exec.replacements != 1 { + t.Fatalf("exec restarted: %d replacements", exec.replacements) + } + if privilege.replacements != 2 || privilege.withdrawals != 1 { + t.Fatalf("privilege %d replacements %d withdrawals", privilege.replacements, privilege.withdrawals) + } +} From 7abe05b893232c32f5215e9eaf5061dce200229c Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:37:35 +0200 Subject: [PATCH 26/54] test(enforcement): a slot move retry arms again after a failed rollback Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- .../common/enforcement/bindings_test.go | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go index d9b20c6384..8cf052426e 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go @@ -20,10 +20,13 @@ type fakeMap struct { frozen bool log *[]string name string - deleteErr error + deleteErr, putErr error } func (m *fakeMap) Put(k, v interface{}) error { + if m.putErr != nil { + return m.putErr + } if m.frozen { return unix.EPERM } @@ -154,6 +157,33 @@ func TestSameSlotRetryArmsAfterFailedArming(t *testing.T) { } } +// A slot move whose arming fails and whose rollback also fails keeps the new +// slot tracked. A later retry with that slot must arm again. +func TestSlotMoveRetryArmsAfterFailedRollback(t *testing.T) { + b, policy, members, _ := newFixture(t, func(Map) error { return nil }) + arms := 0 + b.armExisting = func(Map, string) error { arms++; return nil } + if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}, nil); err != nil { + t.Fatal(err) + } + b.armExisting = func(Map, string) error { + arms++ + members.putErr = errors.New("busy") // the rollback write fails + return errors.New("walk failed") + } + if err := b.Bind(Binding{Container: identity, Slot: 1}); err == nil { + t.Fatal("a failed move must fail the bind") + } + members.putErr = nil + b.armExisting = func(Map, string) error { arms++; return nil } + if err := b.Bind(Binding{Container: identity, Slot: 1}); err != nil || arms != 3 { + t.Fatalf("retry must arm: %v arms=%d", err, arms) + } + if members.data[uint64(42)] != (bindingValue{Slot: 1}) { + t.Fatalf("binding %v", members.data) + } +} + func TestRejectsWrongBindingLayoutAndPolicyErrors(t *testing.T) { b, policy, members, log := newFixture(t, func(Map) error { return nil }) members.max = 8 From b2c4b4f7f53998673540e40ca4aea50a31662d79 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:25:05 +0200 Subject: [PATCH 27/54] feat(sandboxpolicy): bind the pod cgroup before the containers start A new container ran unchecked until its own cgroup was bound, after it started. Bind the pod-level cgroup instead, as soon as the kubelet creates it: container cgroups are created under it, so the privilege gadget judges the first exec of every container. A container binding still decides once it exists. A pod gets a pod binding only when all its declared containers get the same privilege policies. Pod bindings do not arm the tasks already in the pod cgroup, because runc init of a container being created can be there. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- .../gadgets/common/enforcement/bindings.go | 9 +- .../common/enforcement/bindings_test.go | 25 ++++++ pkg/sandboxpolicy/controller.go | 80 ++++++++++++++++-- pkg/sandboxpolicy/igbackend/privilege.go | 83 +++++++++++++++++- pkg/sandboxpolicy/igbackend/privilege_test.go | 67 +++++++++++++++ pkg/sandboxpolicy/integration/bindings.go | 16 ++-- pkg/sandboxpolicy/integration/podcgroup.go | 83 ++++++++++++++++++ .../integration/podcgroup_linux.go | 36 ++++++++ .../integration/podcgroup_test.go | 72 ++++++++++++++++ pkg/sandboxpolicy/integration/service.go | 84 ++++++++++++++++++- pkg/sandboxpolicy/sections.go | 10 +++ pkg/sandboxpolicy/sections_test.go | 30 +++++++ pkg/sandboxpolicy/types.go | 20 +++++ 13 files changed, 596 insertions(+), 19 deletions(-) create mode 100644 pkg/sandboxpolicy/integration/podcgroup.go create mode 100644 pkg/sandboxpolicy/integration/podcgroup_linux.go create mode 100644 pkg/sandboxpolicy/integration/podcgroup_test.go diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings.go b/pkg/ebpf/gadgets/common/enforcement/bindings.go index e578bcaf68..82e1d89cdc 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings.go @@ -33,6 +33,11 @@ type ContainerIdentity = execruntime.ContainerIdentity type Binding struct { Container ContainerIdentity Slot uint32 + // SkipArming binds without arming the tasks already in the cgroup. Use it + // for a parent cgroup bound before its containers start: a container + // runtime helper can be there between its namespace setup and its exec, + // and must not be judged. Tasks are still armed at their own exec. + SkipArming bool } // bindingValue matches struct binding_value in ../bindings.h. @@ -231,8 +236,8 @@ func (b *Bindings) bind(c Binding) error { } // A slot move arms again too: a task that the old slot left unarmed (an // audit slot allows a fork whose child mark failed) must not escape the - // new slot. - if b.armed != nil { + // new slot. A SkipArming binding never arms. + if b.armed != nil && !c.SkipArming { if err := b.armExisting(b.armed, id.CgroupPath); err != nil { if bound && !errors.Is(err, ErrContainerGone) { // Undo the move, so a retry moves and arms again. diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go index 8cf052426e..11ad765734 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go @@ -246,3 +246,28 @@ func TestSlotFaults(t *testing.T) { t.Fatalf("slot 1: %d %v", n, err) } } + +func TestSkipArmingBindsWithoutArming(t *testing.T) { + b, policy, members, _ := newFixture(t, func(Map) error { return nil }) + var armedPaths []string + b.armExisting = func(_ Map, path string) error { armedPaths = append(armedPaths, path); return nil } + parent := ContainerIdentity{Runtime: "parent", ID: "p1", CgroupID: 7, CgroupPath: "/kubepods.slice/kubepods-pod1.slice"} + b.desired[selector(parent)] = Binding{Container: parent, SkipArming: true} + if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}, nil); err != nil { + t.Fatal(err) + } + // Only the initial container binding arms; the replayed parent does not. + if len(armedPaths) != 1 || armedPaths[0] != identity.CgroupPath || members.data[uint64(7)] != (bindingValue{}) { + t.Fatalf("startup armed %v bindings %v", armedPaths, members.data) + } + if err := b.Unbind(parent); err != nil { + t.Fatal(err) + } + if err := b.Bind(Binding{Container: parent, Slot: 1, SkipArming: true}); err != nil || len(armedPaths) != 1 || members.data[uint64(7)] != (bindingValue{Slot: 1}) { + t.Fatalf("live bind: %v armed %v bindings %v", err, armedPaths, members.data) + } + // A slot move of a SkipArming binding does not arm either. + if err := b.Bind(Binding{Container: parent, Slot: 0, SkipArming: true}); err != nil || len(armedPaths) != 1 { + t.Fatalf("slot move: %v armed %v", err, armedPaths) + } +} diff --git a/pkg/sandboxpolicy/controller.go b/pkg/sandboxpolicy/controller.go index 799d8687a5..22e86f1629 100644 --- a/pkg/sandboxpolicy/controller.go +++ b/pkg/sandboxpolicy/controller.go @@ -16,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" ) @@ -179,22 +180,83 @@ func (c *Controller) PrepareScheduled(ctx context.Context, containers []Binding) c.mu.Unlock() return nil } - policies := c.privilegePolicies() + plans := privilegePlans(c.privilegePolicies(), containers) c.mu.Unlock() - var plans []Plan - for _, b := range containers { - var plan Plan + var nonEmpty []Plan + for _, plan := range plans { + if len(plan.Policies) > 0 { + nonEmpty = append(nonEmpty, plan) + } + } + // Outside c.mu: a preparation may start a gadget instance. + return preparer.Prepare(ctx, nonEmpty) +} + +// PodPlans returns the privilege plan of each pod whose declared containers +// (all of them must be given) get the same nonempty set of privilege policies. +// Such a pod can be bound at its pod cgroup before its containers start. A pod +// whose containers get different policies, or none, has no pod plan: a pod +// binding would also judge the containers that the policies do not select. +func (c *Controller) PodPlans(containers []Binding) map[types.UID]Plan { + c.mu.Lock() + if c.configError != nil { + c.mu.Unlock() + return nil + } + plans := privilegePlans(c.privilegePolicies(), containers) + c.mu.Unlock() + pods := map[types.UID]Plan{} + mixed := map[types.UID]bool{} + for i, b := range containers { + plan := plans[i] + if mixed[b.PodUID] { + continue + } + if len(plan.Policies) == 0 { + mixed[b.PodUID] = true + delete(pods, b.PodUID) + continue + } + previous, seen := pods[b.PodUID] + if seen && !samePolicies(previous, plan) { + mixed[b.PodUID] = true + delete(pods, b.PodUID) + continue + } + pods[b.PodUID] = plan + } + return pods +} + +// privilegePlans returns the plan of each container, in order (possibly empty). +func privilegePlans(policies []desiredPolicy, containers []Binding) []Plan { + plans := make([]Plan, len(containers)) + for i, b := range containers { for _, p := range policies { if p.matches(b) { - plan.Policies = append(plan.Policies, CompiledPolicy{Identity: p.identity, Mode: p.spec.Mode, Privilege: p.spec.Privilege.DeepCopy()}) + plans[i].Policies = append(plans[i].Policies, CompiledPolicy{Identity: p.identity, Mode: p.spec.Mode, Privilege: p.spec.Privilege.DeepCopy()}) } } - if len(plan.Policies) > 0 { - plans = append(plans, plan) + } + return plans +} + +// samePolicies compares the policy objects and generations of two plans. +func samePolicies(a, b Plan) bool { + if len(a.Policies) != len(b.Policies) { + return false + } + ids := map[PolicyIdentity]int{} + for _, p := range a.Policies { + ids[p.Identity]++ + } + for _, p := range b.Policies { + if ids[p.Identity] == 0 { + return false } + ids[p.Identity]-- } - // Outside c.mu: a preparation may start a gadget instance. - return preparer.Prepare(ctx, plans) + return true } // HasPrivilegePolicies reports whether any active policy has a privilege section. diff --git a/pkg/sandboxpolicy/igbackend/privilege.go b/pkg/sandboxpolicy/igbackend/privilege.go index 3ddcf692c7..c34fd21f08 100644 --- a/pkg/sandboxpolicy/igbackend/privilege.go +++ b/pkg/sandboxpolicy/igbackend/privilege.go @@ -16,6 +16,7 @@ import ( privpolicy "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/policy" privruntime "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/runtime" "github.com/kubescape/node-agent/pkg/sandboxpolicy" + "k8s.io/apimachinery/pkg/types" ) // PrivilegeEngine starts one node-wide enforce_privilege instance. Start returns @@ -68,6 +69,23 @@ func (p *privilegeBinding) close() error { return p.lease.Close() } +// podBinding is a bound pod cgroup (see sandboxpolicy.PodBinding). +type podBinding struct { + pod sandboxpolicy.PodBinding + combination string +} + +// podRuntime is the runtime name of a pod cgroup identity in the bindings +// controller. No container runtime has this name. +const podRuntime = "kubernetes-pod" + +// nativePod returns the bindings controller identity of a pod cgroup. Tasks +// already in it are not armed: runc init of a container being created can be +// there. +func nativePod(p sandboxpolicy.PodBinding, slot uint32) enforcement.Binding { + return enforcement.Binding{Container: enforcement.ContainerIdentity{Runtime: podRuntime, ID: string(p.PodUID), CgroupID: p.CgroupID, CgroupPath: p.CgroupPath}, Slot: slot, SkipArming: true} +} + // PrivilegeBackend owns ONE enforce_privilege instance per node. // // A slot is one combination of privilege policies; containers with the same @@ -87,6 +105,7 @@ type PrivilegeBackend struct { slots map[string]uint32 // combination -> slot in instance combinations map[string][]sandboxpolicy.CompiledPolicy bindings map[identity]*privilegeBinding + pods map[types.UID]*podBinding prepared map[string][]sandboxpolicy.CompiledPolicy retiring []PrivilegeInstance stats PrivilegeStats @@ -96,6 +115,7 @@ type PrivilegeBackend struct { var ( _ sandboxpolicy.Backend = (*PrivilegeBackend)(nil) _ sandboxpolicy.ProtectionAttester = (*PrivilegeBackend)(nil) + _ sandboxpolicy.PodBinder = (*PrivilegeBackend)(nil) ) // NewPrivilege creates the node backend. validator verifies the live container @@ -104,7 +124,7 @@ func NewPrivilege(engine PrivilegeEngine, validator Validator, prefix string) (* if engine == nil { return nil, fmt.Errorf("privilege engine is required") } - return &PrivilegeBackend{engine: engine, validator: validator, prefix: prefix, slots: map[string]uint32{}, combinations: map[string][]sandboxpolicy.CompiledPolicy{}, bindings: map[identity]*privilegeBinding{}, prepared: map[string][]sandboxpolicy.CompiledPolicy{}}, nil + return &PrivilegeBackend{engine: engine, validator: validator, prefix: prefix, slots: map[string]uint32{}, combinations: map[string][]sandboxpolicy.CompiledPolicy{}, bindings: map[identity]*privilegeBinding{}, pods: map[types.UID]*podBinding{}, prepared: map[string][]sandboxpolicy.CompiledPolicy{}}, nil } // Combination returns the privilege policies of a plan, sorted, and their key. @@ -290,6 +310,58 @@ func (b *PrivilegeBackend) Prepare(ctx context.Context, plans []sandboxpolicy.Pl return errors.Join(b.replaceInstance(ctx, prepared, nil, identity{}, nil), b.retire(ctx)) } +// BindPods binds the given pod cgroups to the slot of their plan and unbinds +// the pod cgroups that are not given. A pod whose slot does not exist yet is +// skipped: Prepare creates the slot, and the next call binds the pod. A pod +// whose plan changes keeps its old slot until the new slot exists, then moves +// in one map write. +func (b *PrivilegeBackend) BindPods(ctx context.Context, pods []sandboxpolicy.PodBinding) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return fmt.Errorf("privilege backend is closed") + } + ready := b.instance != nil && b.instance.Ready() == nil + wanted := make(map[types.UID]sandboxpolicy.PodBinding, len(pods)) + for _, p := range pods { + wanted[p.PodUID] = p + } + var errs []error + for uid, current := range b.pods { + next, ok := wanted[uid] + if ok && next.CgroupID == current.pod.CgroupID && next.CgroupPath == current.pod.CgroupPath { + continue + } + // Not wanted, or a new cgroup for the pod: remove the old identity. + // An instance that is not ready protects nothing. + if ready { + if err := b.instance.Unbind(nativePod(current.pod, 0).Container); err != nil { + errs = append(errs, err) + continue + } + } + delete(b.pods, uid) + } + for uid, p := range wanted { + combination, policies := Combination(p.Plan) + current := b.pods[uid] + if len(policies) == 0 || current != nil && current.combination == combination { + continue + } + slot, ok := b.slots[combination] + if !ok || !ready { + continue + } + if err := b.instance.Bind(nativePod(p, slot)); err != nil { + errs = append(errs, err) + continue + } + b.pods[uid] = &podBinding{pod: p, combination: combination} + logger.L().Info("sandbox privilege pod bind", helpers.String("namespace", p.Namespace), helpers.String("pod", p.PodName)) + } + return errors.Join(append(errs, b.retire(ctx))...) +} + // replaceInstance starts a candidate with every combination in use or prepared // (plus extra), binds every container (plus next, under nextKey), and only then // stops the old instance. On failure the old instance and state stay. @@ -313,6 +385,11 @@ func (b *PrivilegeBackend) replaceInstance(ctx context.Context, prepared, extra wanted[v.combination] = b.combinations[v.combination] } } + for _, p := range b.pods { + if _, ok := wanted[p.combination]; !ok { + wanted[p.combination] = b.combinations[p.combination] + } + } keys := make([]string, 0, len(wanted)) for c := range wanted { keys = append(keys, c) @@ -338,6 +415,9 @@ func (b *PrivilegeBackend) replaceInstance(ctx context.Context, prepared, extra for _, v := range bindings { config.InitialBindings = append(config.InitialBindings, enforcement.Binding{Container: native(v.binding), Slot: slots[v.combination]}) } + for _, p := range b.pods { + config.InitialBindings = append(config.InitialBindings, nativePod(p.pod, slots[p.combination])) + } b.generation++ candidate, err := b.engine.Start(ctx, fmt.Sprintf("%s-privilege-%d", b.prefix, b.generation), config) if err != nil { @@ -385,5 +465,6 @@ func (b *PrivilegeBackend) Close(ctx context.Context) error { errs = append(errs, v.close()) } b.bindings = map[identity]*privilegeBinding{} + b.pods = map[types.UID]*podBinding{} return errors.Join(append(errs, b.retire(ctx))...) } diff --git a/pkg/sandboxpolicy/igbackend/privilege_test.go b/pkg/sandboxpolicy/igbackend/privilege_test.go index d026563edc..af16278ca9 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_test.go +++ b/pkg/sandboxpolicy/igbackend/privilege_test.go @@ -265,3 +265,70 @@ func TestPrivilegeTrackingFaultUnconfirmsAndHeals(t *testing.T) { t.Fatalf("the new instance has no fault: %v", err) } } + +func testPodBinding(uid string, cgroup uint64, plan sandboxpolicy.Plan) sandboxpolicy.PodBinding { + return sandboxpolicy.PodBinding{PodUID: types.UID(uid), Namespace: "ns", PodName: uid, CgroupID: cgroup, CgroupPath: "/kubepods/pod" + uid, Plan: plan} +} + +func TestPrivilegePodBindings(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + p1 := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + p2 := privPlan("r2", privPolicy("p2", api.ActionDeny)) + // No slot yet: the pod waits for Prepare. + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, p1)}); err != nil || len(e.instances) != 0 { + t.Fatalf("no slot: %v instances %d", err, len(e.instances)) + } + if err := b.Prepare(ctx, []sandboxpolicy.Plan{p1}); err != nil { + t.Fatal(err) + } + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, p1)}); err != nil { + t.Fatal(err) + } + slot1, ok := e.instances[0].bound[100] + if !ok { + t.Fatal("the pod cgroup must be bound once its slot exists") + } + // A container of the pod binds to the same slot; the pod binding stays. + if _, err := b.Replace(ctx, privBinding(1), p1); err != nil || e.instances[0].bound[1] != slot1 || len(e.instances) != 1 { + t.Fatalf("container bind: %v %+v", err, e.instances[0].bound) + } + // A new combination replaces the instance; the pod binding is carried. + if err := b.Prepare(ctx, []sandboxpolicy.Plan{p1, p2}); err != nil || len(e.instances) != 2 { + t.Fatalf("prepare p2: %v", err) + } + carried := false + for _, initial := range e.configs[1].InitialBindings { + if initial.Container.CgroupID == 100 { + carried = initial.SkipArming && initial.Container.Runtime == podRuntime && initial.Container.ID == "a" + } + } + if !carried { + t.Fatalf("pod binding not carried without arming: %+v", e.configs[1].InitialBindings) + } + // A changed plan moves the pod to the new slot in place. + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, p2)}); err != nil || e.instances[1].bound[100] == e.instances[1].bound[1] { + t.Fatalf("plan change: %v %+v", err, e.instances[1].bound) + } + // A recreated pod cgroup is a new identity: the old one is removed. + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 101, p2)}); err != nil { + t.Fatal(err) + } + if _, old := e.instances[1].bound[100]; old { + t.Fatal("the old pod cgroup must be unbound") + } + if _, ok := e.instances[1].bound[101]; !ok { + t.Fatal("the new pod cgroup must be bound") + } + // A pod that is not given any more is unbound; containers stay. + if err := b.BindPods(ctx, nil); err != nil { + t.Fatal(err) + } + if _, ok := e.instances[1].bound[101]; ok { + t.Fatal("pod binding must be removed") + } + if _, ok := e.instances[1].bound[1]; !ok { + t.Fatal("the container binding must stay") + } +} diff --git a/pkg/sandboxpolicy/integration/bindings.go b/pkg/sandboxpolicy/integration/bindings.go index 5e5e332515..df5483b803 100644 --- a/pkg/sandboxpolicy/integration/bindings.go +++ b/pkg/sandboxpolicy/integration/bindings.go @@ -13,6 +13,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" ) @@ -63,24 +64,26 @@ func SnapshotSelectedBindingsFromContainers(ctx context.Context, client kubernet // ScheduledContainers returns API metadata (no runtime identity) for every // declared regular, init and ephemeral container of the pods scheduled to this node that -// are not finished. It feeds privilege slot preparation, not binding. -func ScheduledContainers(ctx context.Context, client kubernetes.Interface, nodeName string) ([]sandboxpolicy.Binding, error) { +// are not finished, and the QoS class of each such pod. It feeds privilege slot +// preparation and pod cgroup binding, not container binding. +func ScheduledContainers(ctx context.Context, client kubernetes.Interface, nodeName string) ([]sandboxpolicy.Binding, map[types.UID]corev1.PodQOSClass, error) { if client == nil || nodeName == "" { - return nil, fmt.Errorf("Kubernetes client and node name are required") + return nil, nil, fmt.Errorf("Kubernetes client and node name are required") } pods, err := client.CoreV1().Pods("").List(ctx, metav1.ListOptions{FieldSelector: fields.OneTermEqualSelector("spec.nodeName", nodeName).String()}) if err != nil { - return nil, fmt.Errorf("list node pods: %w", err) + return nil, nil, fmt.Errorf("list node pods: %w", err) } namespaces, err := client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) if err != nil { - return nil, fmt.Errorf("list namespaces: %w", err) + return nil, nil, fmt.Errorf("list namespaces: %w", err) } namespaceLabels := make(map[string]map[string]string, len(namespaces.Items)) for _, ns := range namespaces.Items { namespaceLabels[ns.Name] = ns.Labels } var result []sandboxpolicy.Binding + qos := map[types.UID]corev1.PodQOSClass{} for _, pod := range pods.Items { if pod.Spec.NodeName != nodeName || pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { continue @@ -89,6 +92,7 @@ func ScheduledContainers(ctx context.Context, client kubernetes.Interface, nodeN if !ok { continue } + qos[pod.UID] = pod.Status.QOSClass for _, group := range []struct { kind string containers []corev1.Container @@ -101,7 +105,7 @@ func ScheduledContainers(ctx context.Context, client kubernetes.Interface, nodeN result = append(result, sandboxpolicy.Binding{PodUID: pod.UID, Namespace: pod.Namespace, PodName: pod.Name, ContainerName: c.Name, Kind: "ephemeral", PodLabels: copyLabels(pod.Labels), NamespaceLabels: copyLabels(labels)}) } } - return result, nil + return result, qos, nil } func snapshotBindings(ctx context.Context, client kubernetes.Interface, containers []*containercollection.Container, nodeName string, selector func(sandboxpolicy.Binding) bool) ([]sandboxpolicy.Binding, error) { diff --git a/pkg/sandboxpolicy/integration/podcgroup.go b/pkg/sandboxpolicy/integration/podcgroup.go new file mode 100644 index 0000000000..ccb07d17d4 --- /dev/null +++ b/pkg/sandboxpolicy/integration/podcgroup.go @@ -0,0 +1,83 @@ +package integration + +import ( + "path" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +// podCgroupLayout is how the kubelet of this node names pod cgroups: the +// kubepods cgroup and the cgroup driver. The kubelet creates the pod cgroup +// before the pod sandbox, so a pod cgroup bound at that time covers the first +// exec of every container of the pod. +// +// systemd: /kubepods-burstable.slice/kubepods-burstable-pod.slice +// cgroupfs: /burstable/pod +// +// Guaranteed pods are directly under the root. The root (for example +// /kubepods.slice, or /kubelet.slice/kubelet-kubepods.slice in kind) is read +// from the cgroup path of a running pod container. +type podCgroupLayout struct { + root string + systemd bool +} + +// learnPodCgroupLayout reads the layout from the cgroup v2 path of a container +// of pod uid. It returns false when the path has no pod cgroup of that pod. +func learnPodCgroupLayout(cgroupPath string, uid types.UID) (podCgroupLayout, bool) { + if uid == "" || !path.IsAbs(cgroupPath) || path.Clean(cgroupPath) != cgroupPath { + return podCgroupLayout{}, false + } + parts := strings.Split(cgroupPath, "/") + for i := 2; i < len(parts); i++ { + for _, systemd := range []bool{true, false} { + // QoS levels first: a BestEffort pod cgroup also reads as a + // Guaranteed one under a root named after the QoS level. + for _, qos := range []corev1.PodQOSClass{corev1.PodQOSBestEffort, corev1.PodQOSBurstable, corev1.PodQOSGuaranteed} { + levels := 1 + if qos != corev1.PodQOSGuaranteed { + levels = 2 + } + if i-levels < 1 { + continue + } + layout := podCgroupLayout{root: strings.Join(parts[:i+1-levels], "/"), systemd: systemd} + if layout.path(uid, qos) == strings.Join(parts[:i+1], "/") { + return layout, true + } + } + } + } + return podCgroupLayout{}, false +} + +// path returns the pod cgroup path of a pod, or "" when the QoS class is not known. +func (l podCgroupLayout) path(uid types.UID, qos corev1.PodQOSClass) string { + if l.root == "" || uid == "" || strings.ContainsAny(string(uid), "/ \t\r\n") { + return "" + } + var level string + switch qos { + case corev1.PodQOSGuaranteed: + case corev1.PodQOSBurstable, corev1.PodQOSBestEffort: + level = strings.ToLower(string(qos)) + default: + return "" + } + if !l.systemd { + return path.Join(l.root, level, "pod"+string(uid)) + } + // systemd escapes "-" as "_" in a slice name; "-" separates the levels. + parent := strings.TrimSuffix(path.Base(l.root), ".slice") + if parent == path.Base(l.root) { + return "" + } + dir := l.root + if level != "" { + parent += "-" + level + dir = path.Join(dir, parent+".slice") + } + return path.Join(dir, parent+"-pod"+strings.ReplaceAll(string(uid), "-", "_")+".slice") +} diff --git a/pkg/sandboxpolicy/integration/podcgroup_linux.go b/pkg/sandboxpolicy/integration/podcgroup_linux.go new file mode 100644 index 0000000000..569b7492c8 --- /dev/null +++ b/pkg/sandboxpolicy/integration/podcgroup_linux.go @@ -0,0 +1,36 @@ +package integration + +import ( + "fmt" + "os" + "strings" + + "golang.org/x/sys/unix" +) + +// cgroupID returns the cgroup ID (the inode) of the cgroup v2 directory rel +// under root. Symlinks and paths outside root are refused. +func cgroupID(root, rel string) (uint64, error) { + dir, err := os.Open(root) + if err != nil { + return 0, err + } + defer dir.Close() + fd, err := unix.Openat2(int(dir.Fd()), strings.TrimPrefix(rel, "/"), &unix.OpenHow{Flags: unix.O_RDONLY | unix.O_DIRECTORY | unix.O_CLOEXEC, Resolve: unix.RESOLVE_BENEATH | unix.RESOLVE_NO_SYMLINKS}) + if err != nil { + return 0, err + } + defer unix.Close(fd) + var fs unix.Statfs_t + if err := unix.Fstatfs(fd, &fs); err != nil { + return 0, err + } + if fs.Type != unix.CGROUP2_SUPER_MAGIC { + return 0, fmt.Errorf("%s is not on cgroup v2", rel) + } + var stat unix.Stat_t + if err := unix.Fstat(fd, &stat); err != nil { + return 0, err + } + return stat.Ino, nil +} diff --git a/pkg/sandboxpolicy/integration/podcgroup_test.go b/pkg/sandboxpolicy/integration/podcgroup_test.go new file mode 100644 index 0000000000..1fd31e39cc --- /dev/null +++ b/pkg/sandboxpolicy/integration/podcgroup_test.go @@ -0,0 +1,72 @@ +package integration + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +const testPodUID = types.UID("0f3c2a1b-1111-2222-3333-444455556666") + +func TestPodCgroupLayout(t *testing.T) { + u := "0f3c2a1b_1111_2222_3333_444455556666" + other := types.UID("9e8d7c6b-aaaa-bbbb-cccc-ddddeeeeffff") + o := "9e8d7c6b_aaaa_bbbb_cccc_ddddeeeeffff" + for _, tc := range []struct { + name, container string + want map[corev1.PodQOSClass]string + }{ + {"systemd besteffort", "/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod" + u + ".slice/cri-containerd-abc.scope", map[corev1.PodQOSClass]string{ + corev1.PodQOSBestEffort: "/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod" + o + ".slice", + corev1.PodQOSBurstable: "/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod" + o + ".slice", + corev1.PodQOSGuaranteed: "/kubepods.slice/kubepods-pod" + o + ".slice", + }}, + {"systemd guaranteed, kind root", "/kubelet.slice/kubelet-kubepods.slice/kubelet-kubepods-pod" + u + ".slice/cri-containerd-abc.scope", map[corev1.PodQOSClass]string{ + corev1.PodQOSBurstable: "/kubelet.slice/kubelet-kubepods.slice/kubelet-kubepods-burstable.slice/kubelet-kubepods-burstable-pod" + o + ".slice", + corev1.PodQOSGuaranteed: "/kubelet.slice/kubelet-kubepods.slice/kubelet-kubepods-pod" + o + ".slice", + }}, + {"cgroupfs burstable", "/kubepods/burstable/pod" + string(testPodUID) + "/abc", map[corev1.PodQOSClass]string{ + corev1.PodQOSBestEffort: "/kubepods/besteffort/pod" + string(other), + corev1.PodQOSGuaranteed: "/kubepods/pod" + string(other), + }}, + {"cgroupfs guaranteed", "/kubepods/pod" + string(testPodUID) + "/abc", map[corev1.PodQOSClass]string{ + corev1.PodQOSBurstable: "/kubepods/burstable/pod" + string(other), + }}, + } { + t.Run(tc.name, func(t *testing.T) { + layout, ok := learnPodCgroupLayout(tc.container, testPodUID) + if !ok { + t.Fatalf("layout not learned from %s", tc.container) + } + for qos, want := range tc.want { + if got := layout.path(other, qos); got != want { + t.Errorf("%s: got %s, want %s", qos, got, want) + } + } + if got := layout.path(other, ""); got != "" { + t.Errorf("unknown QoS class must give no path: %s", got) + } + }) + } +} + +func TestPodCgroupLayoutRefusesOtherPaths(t *testing.T) { + for _, path := range []string{ + "/system.slice/docker-abc.scope", + "/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podother.slice/cri-containerd-abc.scope", + "kubepods/pod" + string(testPodUID), + "/kubepods/../pod" + string(testPodUID), + "/pod" + string(testPodUID), + } { + if layout, ok := learnPodCgroupLayout(path, testPodUID); ok { + t.Errorf("%s: unexpected layout %+v", path, layout) + } + } + if _, ok := learnPodCgroupLayout("/kubepods/pod"+string(testPodUID), ""); ok { + t.Error("a container without a pod UID must not give a layout") + } + if got := (podCgroupLayout{root: "/kubepods"}).path("a/b", corev1.PodQOSGuaranteed); got != "" { + t.Errorf("a UID with a separator must give no path: %s", got) + } +} diff --git a/pkg/sandboxpolicy/integration/service.go b/pkg/sandboxpolicy/integration/service.go index 94a1cc9c06..6e0305b007 100644 --- a/pkg/sandboxpolicy/integration/service.go +++ b/pkg/sandboxpolicy/integration/service.go @@ -26,6 +26,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" orasoci "oras.land/oras-go/v2/content/oci" @@ -77,11 +78,28 @@ type Service struct { // often privilege slot preparation lists the node's pods. podsScheduled atomic.Bool lastPrepare time.Time + // scheduled and qos are the containers and QoS classes of the pods + // scheduled to this node, from the last slot preparation. Only the refresh + // loop uses them. + scheduled []sandboxpolicy.Binding + qos map[types.UID]corev1.PodQOSClass + // podPolling is set while a pod with a pod plan has no pod cgroup yet, + // up to podPollUntil (see bindPods). + podPolling bool + podPollUntil time.Time + // podCgroupID returns the cgroup ID of a pod cgroup path (replaced in tests). + podCgroupID func(string) (uint64, error) } // prepareInterval is the longest time between two slot preparations. const prepareInterval = 30 * time.Second +// podPollInterval is how often a scheduled pod is checked for its pod cgroup, +// for up to prepareInterval after it is scheduled. The kubelet creates the pod +// cgroup before the pod sandbox, so the pod cgroup is bound before the first +// container starts. +const podPollInterval = 50 * time.Millisecond + // NewService prepares the public watcher/exporter/native runtime bridge. It does // not authorize any cgroup or start enforcement until Start is called. func NewService(ctx context.Context, options ServiceOptions) (*Service, error) { @@ -101,6 +119,8 @@ func NewService(ctx context.Context, options ServiceOptions) (*Service, error) { return nil, fmt.Errorf("sandbox native runtime: %w", err) } s := &Service{options: options, bindings: map[string]sandboxpolicy.Binding{}, closeRuntime: r.Close, done: make(chan struct{}), wake: make(chan struct{}, 1)} + cgroupRoot := filepath.Join(host.HostRoot, "sys/fs/cgroup") + s.podCgroupID = func(path string) (uint64, error) { return cgroupID(cgroupRoot, path) } s.exporter, err = sandboxpolicy.NewVerdictExporter(sandboxpolicy.ExporterOptions{Capacity: 1024, Receivers: options.Receivers, Enricher: options.Enricher, ResolveBinding: s.resolveBinding}) if err != nil { _ = r.Close() @@ -241,6 +261,10 @@ drain: ticker := time.NewTicker(5 * time.Second) defer ticker.Stop() for { + var poll <-chan time.Time + if s.podPolling { + poll = time.After(podPollInterval) + } select { case <-runCtx.Done(): return @@ -248,6 +272,8 @@ drain: s.refresh(runCtx) case <-s.wake: s.refresh(runCtx) + case <-poll: + s.bindPods(runCtx) } } }() @@ -260,14 +286,17 @@ func (s *Service) refresh(ctx context.Context) { if s.options.PrivilegeImageName != "" && s.options.Kubernetes != nil && s.Controller.HasPrivilegePolicies() && (s.podsScheduled.Swap(false) || time.Since(s.lastPrepare) >= prepareInterval) { s.lastPrepare = time.Now() - scheduled, err := ScheduledContainers(ctx, s.options.Kubernetes, s.options.NodeName) + scheduled, qos, err := ScheduledContainers(ctx, s.options.Kubernetes, s.options.NodeName) if err == nil { + s.scheduled, s.qos, s.podPollUntil = scheduled, qos, s.lastPrepare.Add(prepareInterval) err = s.Controller.PrepareScheduled(ctx, scheduled) } if err != nil { s.report(fmt.Errorf("privilege slot preparation: %w", err)) } } + // Every refresh, so a policy change also reaches the pod bindings. + s.bindPods(ctx) // Configuration and selected-runtime discovery share one commit barrier. A // candidate cannot report Ready between its relist and a failed binding list. err := s.Controller.Refresh(ctx, func(ctx context.Context, selected func(sandboxpolicy.Binding) bool) ([]sandboxpolicy.Binding, error) { @@ -299,6 +328,59 @@ func (s *Service) refresh(ctx context.Context) { } } +// bindPods binds the pod cgroup of every scheduled pod that has a pod plan +// (see Controller.PodPlans) and unbinds the others. It uses the pods of the +// last slot preparation and makes no API call. A pod whose cgroup does not +// exist yet sets podPolling, so it is checked again soon. +func (s *Service) bindPods(ctx context.Context) { + binder, ok := s.backend.(sandboxpolicy.PodBinder) + if !ok || s.Controller == nil { + return + } + plans := s.Controller.PodPlans(s.scheduled) + var pods []sandboxpolicy.PodBinding + missing := false + if len(plans) > 0 { + layout, known := s.podCgroupLayout() + for _, c := range s.scheduled { + plan, ok := plans[c.PodUID] + if !ok { + continue + } + delete(plans, c.PodUID) + path := layout.path(c.PodUID, s.qos[c.PodUID]) + if !known || path == "" || s.podCgroupID == nil { + continue + } + id, err := s.podCgroupID(path) + if err != nil || id == 0 { + missing = true + continue + } + pods = append(pods, sandboxpolicy.PodBinding{PodUID: c.PodUID, Namespace: c.Namespace, PodName: c.PodName, CgroupID: id, CgroupPath: path, Plan: plan}) + } + } + s.podPolling = missing && time.Now().Before(s.podPollUntil) + if err := binder.BindPods(ctx, pods); err != nil { + s.report(fmt.Errorf("privilege pod binding: %w", err)) + } +} + +// podCgroupLayout reads the pod cgroup layout from a running pod container. +func (s *Service) podCgroupLayout() (podCgroupLayout, bool) { + var layout podCgroupLayout + found := false + if s.options.Containers == nil { + return layout, false + } + s.options.Containers.ContainerRange(func(c *containercollection.Container) { + if !found && c != nil { + layout, found = learnPodCgroupLayout(c.CgroupV2, types.UID(c.K8s.PodUID)) + } + }) + return layout, found +} + // ContainerCallback is deliberately nonblocking: event delivery must never hold // discovery locks while a gadget is loading or Kubernetes is unavailable. func (s *Service) ContainerCallback(containercollection.PubSubEvent) { diff --git a/pkg/sandboxpolicy/sections.go b/pkg/sandboxpolicy/sections.go index e64ce02da7..2d7a547d76 100644 --- a/pkg/sandboxpolicy/sections.go +++ b/pkg/sandboxpolicy/sections.go @@ -79,6 +79,16 @@ func (s *SectionBackend) Prepare(ctx context.Context, plans []Plan) error { return preparer.Prepare(ctx, plans) } +// BindPods passes the pod bindings to a privilege backend that can bind pod +// cgroups. Pod bindings exist only for the privilege section. +func (s *SectionBackend) BindPods(ctx context.Context, pods []PodBinding) error { + binder, ok := s.privilege.(PodBinder) + if !ok { + return nil + } + return binder.BindPods(ctx, pods) +} + // Exec returns the exec section backend. func (s *SectionBackend) Exec() Backend { return s.exec } diff --git a/pkg/sandboxpolicy/sections_test.go b/pkg/sandboxpolicy/sections_test.go index 86b8ad897e..77b72edf86 100644 --- a/pkg/sandboxpolicy/sections_test.go +++ b/pkg/sandboxpolicy/sections_test.go @@ -7,6 +7,7 @@ import ( api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) func privilegeSection(def api.Action) *api.PrivilegeSection { @@ -300,3 +301,32 @@ func TestSectionBackendSeparatePrivilegePolicyKeepsExecRevision(t *testing.T) { t.Fatalf("privilege %d replacements %d withdrawals", privilege.replacements, privilege.withdrawals) } } + +func TestControllerPodPlans(t *testing.T) { + ctx := context.Background() + all := policy("all", "ns") + all.Spec.Privilege = privilegeSection(api.ActionDeny) + app := policy("app-only", "ns") + app.Spec.Containers = []string{"app"} + app.Spec.Privilege = privilegeSection(api.ActionAllow) + c, _, _, _ := setup(t, all, app) + if err := c.InitialSync(ctx); err != nil { + t.Fatal(err) + } + container := func(pod types.UID, name, kind string) Binding { + b := binding() + b.ContainerID, b.CgroupID, b.PodUID, b.ContainerName, b.Kind = "", 0, pod, name, kind + return b + } + elsewhere := container("other-ns", "web", "regular") + elsewhere.Namespace = "elsewhere" + plans := c.PodPlans([]Binding{ + container("uniform", "web", "regular"), container("uniform", "db", "regular"), + container("init", "web", "regular"), container("init", "setup", "init"), + container("mixed", "web", "regular"), container("mixed", "app", "regular"), + elsewhere, + }) + if len(plans) != 1 || len(plans["uniform"].Policies) != 1 || plans["uniform"].Policies[0].Identity.Name != "all" { + t.Fatalf("only the pod whose containers all get the same policies has a pod plan: %+v", plans) + } +} diff --git a/pkg/sandboxpolicy/types.go b/pkg/sandboxpolicy/types.go index 51c1865d17..649d452fa4 100644 --- a/pkg/sandboxpolicy/types.go +++ b/pkg/sandboxpolicy/types.go @@ -110,6 +110,26 @@ type SlotPreparer interface { Prepare(context.Context, []Plan) error } +// PodBinding places the pod-level cgroup of a pod scheduled to this node in +// the plan that every container of the pod gets. Container cgroups are created +// under the pod cgroup, so a container is judged from its first exec, before +// its own binding exists. The container binding still decides once it exists. +type PodBinding struct { + PodUID types.UID + Namespace string + PodName string + CgroupID uint64 + CgroupPath string + Plan Plan +} + +// PodBinder optionally binds pod cgroups. BindPods receives the complete set: +// a pod binding that is not in it is removed. Like SlotPreparer, it is a hint +// that changes no container binding. +type PodBinder interface { + BindPods(context.Context, []PodBinding) error +} + // SourceResolver reads ContainerProfile execs in the selected pod's namespace. // It must return an error for missing/incomplete profiles; empty successful results // mean a known complete profile with no execs. Rules retain their predicates. From d3ad5d573fb699363819ddd55302a284f9191243 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:38:30 +0200 Subject: [PATCH 28/54] fix(sandboxpolicy): learn of scheduled pods without the watcher cooldown The generic watcher delivers a pod event only after the pod has had no update for 5 s, which is later than the start of its containers. Use a dedicated pod informer for the Pending-pod hint, so the slot and the pod cgroup binding are ready before the first container starts. Keep the pod bindings when the policies are not known (configuration error) instead of removing them. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/enforce_privilege/README.md | 5 ++ pkg/sandboxpolicy/controller.go | 7 +-- pkg/sandboxpolicy/integration/service.go | 53 +++++++++++++------- pkg/sandboxpolicy/sections_test.go | 4 +- 4 files changed, 46 insertions(+), 23 deletions(-) diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index a20dd5ebd4..ebbadf6d96 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -108,6 +108,11 @@ policy: zero and arms the running tasks again. Audit-only faults do not count there. - **Scope.** A bound cgroup also covers up to 31 levels of child cgroups. This is by design and differs from enforce_exec, which binds the exact cgroup only. + The nearest bound cgroup decides. So an owner can bind a parent cgroup (for + example the cgroup of a group of containers) before they start, with + `SkipArming`, and a container is judged from its first exec. The tasks + already in the parent are not armed, also not at a slot move or in a new + instance, because a runtime helper can be there during its setup. - **Capabilities.** A `capable()` call is judged only when the task uses its own credentials (overlayfs copy-up uses the mounter's) and the call is not a no-audit probe. diff --git a/pkg/sandboxpolicy/controller.go b/pkg/sandboxpolicy/controller.go index 22e86f1629..c91f39063b 100644 --- a/pkg/sandboxpolicy/controller.go +++ b/pkg/sandboxpolicy/controller.go @@ -197,11 +197,12 @@ func (c *Controller) PrepareScheduled(ctx context.Context, containers []Binding) // Such a pod can be bound at its pod cgroup before its containers start. A pod // whose containers get different policies, or none, has no pod plan: a pod // binding would also judge the containers that the policies do not select. -func (c *Controller) PodPlans(containers []Binding) map[types.UID]Plan { +// It returns false when the policies are not known (a configuration error). +func (c *Controller) PodPlans(containers []Binding) (map[types.UID]Plan, bool) { c.mu.Lock() if c.configError != nil { c.mu.Unlock() - return nil + return nil, false } plans := privilegePlans(c.privilegePolicies(), containers) c.mu.Unlock() @@ -225,7 +226,7 @@ func (c *Controller) PodPlans(containers []Binding) map[types.UID]Plan { } pods[b.PodUID] = plan } - return pods + return pods, true } // privilegePlans returns the plan of each container, in order (possibly empty). diff --git a/pkg/sandboxpolicy/integration/service.go b/pkg/sandboxpolicy/integration/service.go index 6e0305b007..7074ea2852 100644 --- a/pkg/sandboxpolicy/integration/service.go +++ b/pkg/sandboxpolicy/integration/service.go @@ -24,11 +24,13 @@ import ( api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" + "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" orasoci "oras.land/oras-go/v2/content/oci" ) @@ -88,7 +90,8 @@ type Service struct { podPolling bool podPollUntil time.Time // podCgroupID returns the cgroup ID of a pod cgroup path (replaced in tests). - podCgroupID func(string) (uint64, error) + podCgroupID func(string) (uint64, error) + podInformers informers.SharedInformerFactory } // prepareInterval is the longest time between two slot preparations. @@ -188,15 +191,8 @@ func NewService(ctx context.Context, options ServiceOptions) (*Service, error) { // Watch events are hints only. Generic watcher queues may deliver old updates // after a relist/deletion; applying their payload could resurrect removed policy. // Refresh always reads authoritative configuration and cannot revive queued UIDs. -// WatchResources adds this node's pods, so a scheduled pod can get its privilege -// slot before its containers start. func (s *Service) WatchResources() []watcher.WatchResource { - resources := s.Controller.WatchResources() - if s.options.PrivilegeImageName == "" { - return resources - } - pods := watcher.NewWatchResource(schema.GroupVersionResource{Version: "v1", Resource: "pods"}, metav1.ListOptions{FieldSelector: "spec.nodeName=" + s.options.NodeName}) - return append(resources, pods) + return s.Controller.WatchResources() } func (s *Service) AddHandler(_ context.Context, obj runtime.Object) { s.configurationHint(obj) } func (s *Service) ModifyHandler(_ context.Context, obj runtime.Object) { s.configurationHint(obj) } @@ -205,19 +201,33 @@ func (s *Service) configurationHint(obj runtime.Object) { if obj == nil { return } - if pod, ok := obj.(*corev1.Pod); ok { - if s.options.PrivilegeImageName != "" && pod.Spec.NodeName == s.options.NodeName && pod.Status.Phase == corev1.PodPending { - s.podsScheduled.Store(true) - s.ContainerCallback(containercollection.PubSubEvent{}) - } - return - } switch obj.GetObjectKind().GroupVersionKind().Kind { case "SandboxPolicy", "ClusterSandboxPolicy", "SandboxPolicyClasses": s.ContainerCallback(containercollection.PubSubEvent{}) } } +// watchPods hints a slot preparation for each pod scheduled to this node while +// it is Pending. It has its own informer: the generic watcher delivers an +// event only after the object has had no update for 5 s, which is later than +// the start of the pod's containers. +func (s *Service) watchPods(ctx context.Context) { + if s.options.PrivilegeImageName == "" || s.options.Kubernetes == nil { + return + } + s.podInformers = informers.NewSharedInformerFactoryWithOptions(s.options.Kubernetes, 0, informers.WithTweakListOptions(func(o *metav1.ListOptions) { + o.FieldSelector = fields.OneTermEqualSelector("spec.nodeName", s.options.NodeName).String() + })) + hint := func(obj any) { + if pod, ok := obj.(*corev1.Pod); ok && pod.Spec.NodeName == s.options.NodeName && pod.Status.Phase == corev1.PodPending { + s.podsScheduled.Store(true) + s.ContainerCallback(containercollection.PubSubEvent{}) + } + } + _, _ = s.podInformers.Core().V1().Pods().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{AddFunc: hint, UpdateFunc: func(_, obj any) { hint(obj) }}) + s.podInformers.Start(ctx.Done()) +} + // Start is called after container discovery is running and before the generic // dynamic watcher starts. Relisting repairs watch disconnections and re-resolves // source profiles/selector labels. Failed snapshots never become an empty list. @@ -238,6 +248,7 @@ func (s *Service) Start(ctx context.Context) error { } s.mu.Unlock() go s.exporter.Run(runCtx) + s.watchPods(runCtx) // Drain observed startup churn into the readiness barrier before handing // work to the background loop. Persistent churn stays explicitly NotReady. drain: @@ -337,7 +348,10 @@ func (s *Service) bindPods(ctx context.Context) { if !ok || s.Controller == nil { return } - plans := s.Controller.PodPlans(s.scheduled) + plans, ok := s.Controller.PodPlans(s.scheduled) + if !ok { + return // the policies are not known: keep the pod bindings + } var pods []sandboxpolicy.PodBinding missing := false if len(plans) > 0 { @@ -427,6 +441,9 @@ func (s *Service) Close(ctx context.Context) error { case <-ctx.Done(): return ctx.Err() } + if s.podInformers != nil { + s.podInformers.Shutdown() + } } if !s.backendClosed { if err := s.backend.Close(ctx); err != nil { diff --git a/pkg/sandboxpolicy/sections_test.go b/pkg/sandboxpolicy/sections_test.go index 77b72edf86..071acf5e82 100644 --- a/pkg/sandboxpolicy/sections_test.go +++ b/pkg/sandboxpolicy/sections_test.go @@ -320,13 +320,13 @@ func TestControllerPodPlans(t *testing.T) { } elsewhere := container("other-ns", "web", "regular") elsewhere.Namespace = "elsewhere" - plans := c.PodPlans([]Binding{ + plans, ok := c.PodPlans([]Binding{ container("uniform", "web", "regular"), container("uniform", "db", "regular"), container("init", "web", "regular"), container("init", "setup", "init"), container("mixed", "web", "regular"), container("mixed", "app", "regular"), elsewhere, }) - if len(plans) != 1 || len(plans["uniform"].Policies) != 1 || plans["uniform"].Policies[0].Identity.Name != "all" { + if !ok || len(plans) != 1 || len(plans["uniform"].Policies) != 1 || plans["uniform"].Policies[0].Identity.Name != "all" { t.Fatalf("only the pod whose containers all get the same policies has a pod plan: %+v", plans) } } From eb2926c0cbd52e1666161d9a34bff4a052cf4849 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 3 Nov 2026 22:11:50 +0200 Subject: [PATCH 29/54] fix(sandboxpolicy): keep pod bindings in step with pod and policy changes Address review findings on the pod cgroup binding: - read the node's pods from the informer cache on every refresh, and wake on label and ephemeral container changes, so a deselected pod or an added debug container does not stay under a pod binding; - unbind a pod whose new combination has no slot yet, so a removed or edited policy never stays active at pod level, and prepare slots on policy changes; - poll for a pod cgroup for at most prepareInterval per pod, and skip mirror pods, whose pod cgroup is named after the kubelet's own UID; - skip a pod cgroup that a container uses as its own cgroup; - drop pod bindings in the fallback replacement, so they never block a container; - create the pod informer in NewService. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/igbackend/privilege.go | 39 +++-- pkg/sandboxpolicy/igbackend/privilege_test.go | 30 ++++ pkg/sandboxpolicy/integration/bindings.go | 36 +++-- pkg/sandboxpolicy/integration/service.go | 149 ++++++++++++------ pkg/sandboxpolicy/integration/service_test.go | 77 +++++++++ 5 files changed, 259 insertions(+), 72 deletions(-) diff --git a/pkg/sandboxpolicy/igbackend/privilege.go b/pkg/sandboxpolicy/igbackend/privilege.go index c34fd21f08..a03e3b8e24 100644 --- a/pkg/sandboxpolicy/igbackend/privilege.go +++ b/pkg/sandboxpolicy/igbackend/privilege.go @@ -224,10 +224,11 @@ func (b *PrivilegeBackend) Replace(ctx context.Context, binding sandboxpolicy.Bi return commit() } extra := map[string][]sandboxpolicy.CompiledPolicy{combination: policies} - err := b.replaceInstance(ctx, b.prepared, extra, k, next) - if err != nil && len(b.prepared) > 0 { - // The prepared set may not fit; never let it block a real container. - err = b.replaceInstance(ctx, nil, extra, k, next) + err := b.replaceInstance(ctx, b.prepared, extra, k, next, true) + if err != nil && (len(b.prepared) > 0 || len(b.pods) > 0) { + // The prepared set and the pod bindings may not fit; never let them + // block a real container. BindPods binds the pods again later. + err = b.replaceInstance(ctx, nil, extra, k, next, false) } if err != nil { return fail(err) @@ -307,14 +308,15 @@ func (b *PrivilegeBackend) Prepare(ctx context.Context, plans []sandboxpolicy.Pl return b.retire(ctx) } // The prepared set is kept only when the instance with it is running. - return errors.Join(b.replaceInstance(ctx, prepared, nil, identity{}, nil), b.retire(ctx)) + return errors.Join(b.replaceInstance(ctx, prepared, nil, identity{}, nil, true), b.retire(ctx)) } // BindPods binds the given pod cgroups to the slot of their plan and unbinds // the pod cgroups that are not given. A pod whose slot does not exist yet is // skipped: Prepare creates the slot, and the next call binds the pod. A pod -// whose plan changes keeps its old slot until the new slot exists, then moves -// in one map write. +// whose plan changes moves to the new slot in one map write; when the new slot +// does not exist yet, the pod is unbound, so a removed or edited policy never +// stays active at pod level. func (b *PrivilegeBackend) BindPods(ctx context.Context, pods []sandboxpolicy.PodBinding) error { b.mu.Lock() defer b.mu.Unlock() @@ -330,7 +332,10 @@ func (b *PrivilegeBackend) BindPods(ctx context.Context, pods []sandboxpolicy.Po for uid, current := range b.pods { next, ok := wanted[uid] if ok && next.CgroupID == current.pod.CgroupID && next.CgroupPath == current.pod.CgroupPath { - continue + combination, _ := Combination(next.Plan) + if _, slot := b.slots[combination]; slot || combination == current.combination { + continue + } } // Not wanted, or a new cgroup for the pod: remove the old identity. // An instance that is not ready protects nothing. @@ -363,9 +368,15 @@ func (b *PrivilegeBackend) BindPods(ctx context.Context, pods []sandboxpolicy.Po } // replaceInstance starts a candidate with every combination in use or prepared -// (plus extra), binds every container (plus next, under nextKey), and only then -// stops the old instance. On failure the old instance and state stay. -func (b *PrivilegeBackend) replaceInstance(ctx context.Context, prepared, extra map[string][]sandboxpolicy.CompiledPolicy, nextKey identity, next *privilegeBinding) error { +// (plus extra), binds every container (plus next, under nextKey) and, with +// withPods, every pod cgroup, and only then stops the old instance. Without +// withPods, the pod bindings are dropped. On failure the old instance and +// state stay. +func (b *PrivilegeBackend) replaceInstance(ctx context.Context, prepared, extra map[string][]sandboxpolicy.CompiledPolicy, nextKey identity, next *privilegeBinding, withPods bool) error { + pods := b.pods + if !withPods { + pods = map[types.UID]*podBinding{} + } bindings := map[identity]*privilegeBinding{} for k, v := range b.bindings { bindings[k] = v @@ -385,7 +396,7 @@ func (b *PrivilegeBackend) replaceInstance(ctx context.Context, prepared, extra wanted[v.combination] = b.combinations[v.combination] } } - for _, p := range b.pods { + for _, p := range pods { if _, ok := wanted[p.combination]; !ok { wanted[p.combination] = b.combinations[p.combination] } @@ -415,7 +426,7 @@ func (b *PrivilegeBackend) replaceInstance(ctx context.Context, prepared, extra for _, v := range bindings { config.InitialBindings = append(config.InitialBindings, enforcement.Binding{Container: native(v.binding), Slot: slots[v.combination]}) } - for _, p := range b.pods { + for _, p := range pods { config.InitialBindings = append(config.InitialBindings, nativePod(p.pod, slots[p.combination])) } b.generation++ @@ -433,7 +444,7 @@ func (b *PrivilegeBackend) replaceInstance(ctx context.Context, prepared, extra if b.instance != nil { b.retiring = append(b.retiring, b.instance) } - b.instance, b.slots, b.combinations, b.bindings, b.prepared = candidate, slots, wanted, bindings, prepared + b.instance, b.slots, b.combinations, b.bindings, b.prepared, b.pods = candidate, slots, wanted, bindings, prepared, pods b.stats.Replacements++ return nil } diff --git a/pkg/sandboxpolicy/igbackend/privilege_test.go b/pkg/sandboxpolicy/igbackend/privilege_test.go index af16278ca9..9c84662219 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_test.go +++ b/pkg/sandboxpolicy/igbackend/privilege_test.go @@ -332,3 +332,33 @@ func TestPrivilegePodBindings(t *testing.T) { t.Fatal("the container binding must stay") } } + +func TestPrivilegePodUnboundWhenNewSlotIsMissing(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + p1 := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + if err := b.Prepare(ctx, []sandboxpolicy.Plan{p1}); err != nil { + t.Fatal(err) + } + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, p1)}); err != nil { + t.Fatal(err) + } + // An edited policy without a slot yet: the old policy must not stay. + edited := privPlan("r2", privPolicy("p1", api.ActionDeny)) + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, edited)}); err != nil { + t.Fatal(err) + } + if _, ok := e.instances[0].bound[100]; ok { + t.Fatal("a pod whose new slot is missing must be unbound") + } + if err := b.Prepare(ctx, []sandboxpolicy.Plan{edited}); err != nil { + t.Fatal(err) + } + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, edited)}); err != nil { + t.Fatal(err) + } + if _, ok := e.instances[len(e.instances)-1].bound[100]; !ok { + t.Fatal("the pod must be bound once the new slot exists") + } +} diff --git a/pkg/sandboxpolicy/integration/bindings.go b/pkg/sandboxpolicy/integration/bindings.go index df5483b803..92925fe959 100644 --- a/pkg/sandboxpolicy/integration/bindings.go +++ b/pkg/sandboxpolicy/integration/bindings.go @@ -63,36 +63,52 @@ func SnapshotSelectedBindingsFromContainers(ctx context.Context, client kubernet } // ScheduledContainers returns API metadata (no runtime identity) for every -// declared regular, init and ephemeral container of the pods scheduled to this node that -// are not finished, and the QoS class of each such pod. It feeds privilege slot +// declared regular, init and ephemeral container of the pods scheduled to this +// node that are not finished, the QoS class of each such pod (see +// scheduledContainers) and the namespace labels. It feeds privilege slot // preparation and pod cgroup binding, not container binding. -func ScheduledContainers(ctx context.Context, client kubernetes.Interface, nodeName string) ([]sandboxpolicy.Binding, map[types.UID]corev1.PodQOSClass, error) { +func ScheduledContainers(ctx context.Context, client kubernetes.Interface, nodeName string) ([]sandboxpolicy.Binding, map[types.UID]corev1.PodQOSClass, map[string]map[string]string, error) { if client == nil || nodeName == "" { - return nil, nil, fmt.Errorf("Kubernetes client and node name are required") + return nil, nil, nil, fmt.Errorf("Kubernetes client and node name are required") } pods, err := client.CoreV1().Pods("").List(ctx, metav1.ListOptions{FieldSelector: fields.OneTermEqualSelector("spec.nodeName", nodeName).String()}) if err != nil { - return nil, nil, fmt.Errorf("list node pods: %w", err) + return nil, nil, nil, fmt.Errorf("list node pods: %w", err) } namespaces, err := client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) if err != nil { - return nil, nil, fmt.Errorf("list namespaces: %w", err) + return nil, nil, nil, fmt.Errorf("list namespaces: %w", err) } namespaceLabels := make(map[string]map[string]string, len(namespaces.Items)) for _, ns := range namespaces.Items { namespaceLabels[ns.Name] = ns.Labels } + list := make([]*corev1.Pod, len(pods.Items)) + for i := range pods.Items { + list[i] = &pods.Items[i] + } + result, qos := scheduledContainers(list, namespaceLabels, nodeName) + return result, qos, namespaceLabels, nil +} + +// scheduledContainers is ScheduledContainers for a given pod list. A pod of a +// namespace without labels is skipped. A mirror (static) pod has no QoS class +// in the result: its pod cgroup is named after the kubelet's own UID, not the +// API UID, so it gets no pod binding. +func scheduledContainers(pods []*corev1.Pod, namespaceLabels map[string]map[string]string, nodeName string) ([]sandboxpolicy.Binding, map[types.UID]corev1.PodQOSClass) { var result []sandboxpolicy.Binding qos := map[types.UID]corev1.PodQOSClass{} - for _, pod := range pods.Items { - if pod.Spec.NodeName != nodeName || pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { + for _, pod := range pods { + if pod == nil || pod.Spec.NodeName != nodeName || pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { continue } labels, ok := namespaceLabels[pod.Namespace] if !ok { continue } - qos[pod.UID] = pod.Status.QOSClass + if _, mirror := pod.Annotations[corev1.MirrorPodAnnotationKey]; !mirror { + qos[pod.UID] = pod.Status.QOSClass + } for _, group := range []struct { kind string containers []corev1.Container @@ -105,7 +121,7 @@ func ScheduledContainers(ctx context.Context, client kubernetes.Interface, nodeN result = append(result, sandboxpolicy.Binding{PodUID: pod.UID, Namespace: pod.Namespace, PodName: pod.Name, ContainerName: c.Name, Kind: "ephemeral", PodLabels: copyLabels(pod.Labels), NamespaceLabels: copyLabels(labels)}) } } - return result, qos, nil + return result, qos } func snapshotBindings(ctx context.Context, client kubernetes.Interface, containers []*containercollection.Container, nodeName string, selector func(sandboxpolicy.Binding) bool) ([]sandboxpolicy.Binding, error) { diff --git a/pkg/sandboxpolicy/integration/service.go b/pkg/sandboxpolicy/integration/service.go index 7074ea2852..ce7dba7bdc 100644 --- a/pkg/sandboxpolicy/integration/service.go +++ b/pkg/sandboxpolicy/integration/service.go @@ -3,6 +3,7 @@ package integration import ( "context" "fmt" + stdmaps "maps" "path/filepath" "sync" "sync/atomic" @@ -25,6 +26,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" @@ -76,22 +78,25 @@ type Service struct { wake chan struct{} inputEpoch atomic.Uint64 snapshotEpoch atomic.Uint64 - // podsScheduled is set by a Pending-pod event; lastPrepare bounds how - // often privilege slot preparation lists the node's pods. + // podsScheduled is set by a Pending-pod or policy event; lastPrepare + // bounds how often privilege slot preparation lists the node's pods. podsScheduled atomic.Bool lastPrepare time.Time - // scheduled and qos are the containers and QoS classes of the pods - // scheduled to this node, from the last slot preparation. Only the refresh - // loop uses them. - scheduled []sandboxpolicy.Binding - qos map[types.UID]corev1.PodQOSClass - // podPolling is set while a pod with a pod plan has no pod cgroup yet, - // up to podPollUntil (see bindPods). + // The fields below are used only by the refresh loop. + // namespaceLabels are from the last slot preparation. scheduled and qos + // are the containers and QoS classes of the node's pods from the last + // preparation, used when there is no pod informer. + namespaceLabels map[string]map[string]string + scheduled []sandboxpolicy.Binding + qos map[types.UID]corev1.PodQOSClass + // podFirstSeen is when a pod with a pod plan was first seen. podPolling is + // set while such a pod has no pod cgroup yet, for up to prepareInterval. + podFirstSeen map[types.UID]time.Time podPolling bool - podPollUntil time.Time // podCgroupID returns the cgroup ID of a pod cgroup path (replaced in tests). - podCgroupID func(string) (uint64, error) - podInformers informers.SharedInformerFactory + podCgroupID func(string) (uint64, error) + // pods is the node's pod informer (nil without a privilege image). + pods informers.SharedInformerFactory } // prepareInterval is the longest time between two slot preparations. @@ -124,6 +129,12 @@ func NewService(ctx context.Context, options ServiceOptions) (*Service, error) { s := &Service{options: options, bindings: map[string]sandboxpolicy.Binding{}, closeRuntime: r.Close, done: make(chan struct{}), wake: make(chan struct{}, 1)} cgroupRoot := filepath.Join(host.HostRoot, "sys/fs/cgroup") s.podCgroupID = func(path string) (uint64, error) { return cgroupID(cgroupRoot, path) } + if options.PrivilegeImageName != "" { + s.pods = informers.NewSharedInformerFactoryWithOptions(options.Kubernetes, 0, informers.WithTweakListOptions(func(o *metav1.ListOptions) { + o.FieldSelector = fields.OneTermEqualSelector("spec.nodeName", options.NodeName).String() + })) + s.watchPods() + } s.exporter, err = sandboxpolicy.NewVerdictExporter(sandboxpolicy.ExporterOptions{Capacity: 1024, Receivers: options.Receivers, Enricher: options.Enricher, ResolveBinding: s.resolveBinding}) if err != nil { _ = r.Close() @@ -203,29 +214,39 @@ func (s *Service) configurationHint(obj runtime.Object) { } switch obj.GetObjectKind().GroupVersionKind().Kind { case "SandboxPolicy", "ClusterSandboxPolicy", "SandboxPolicyClasses": + // A policy change can need a new slot for the pod bindings. + s.podsScheduled.Store(true) s.ContainerCallback(containercollection.PubSubEvent{}) } } -// watchPods hints a slot preparation for each pod scheduled to this node while -// it is Pending. It has its own informer: the generic watcher delivers an -// event only after the object has had no update for 5 s, which is later than -// the start of the pod's containers. -func (s *Service) watchPods(ctx context.Context) { - if s.options.PrivilegeImageName == "" || s.options.Kubernetes == nil { - return - } - s.podInformers = informers.NewSharedInformerFactoryWithOptions(s.options.Kubernetes, 0, informers.WithTweakListOptions(func(o *metav1.ListOptions) { - o.FieldSelector = fields.OneTermEqualSelector("spec.nodeName", s.options.NodeName).String() - })) - hint := func(obj any) { - if pod, ok := obj.(*corev1.Pod); ok && pod.Spec.NodeName == s.options.NodeName && pod.Status.Phase == corev1.PodPending { +// watchPods wakes the refresh loop on changes of the node's pods. A Pending pod +// also asks for a slot preparation. A label or ephemeral container change can +// change the pod plan, so the pod bindings are computed again. It is a +// dedicated informer: the generic watcher delivers an event only after the +// object has had no update for 5 s, which is later than the start of the +// pod's containers. +func (s *Service) watchPods() { + hint := func(old, obj any) { + pod, ok := obj.(*corev1.Pod) + if !ok { + s.ContainerCallback(containercollection.PubSubEvent{}) // deleted + return + } + if pod.Status.Phase == corev1.PodPending { s.podsScheduled.Store(true) s.ContainerCallback(containercollection.PubSubEvent{}) + return + } + if before, ok := old.(*corev1.Pod); !ok || !stdmaps.Equal(before.Labels, pod.Labels) || len(before.Spec.EphemeralContainers) != len(pod.Spec.EphemeralContainers) { + s.ContainerCallback(containercollection.PubSubEvent{}) } } - _, _ = s.podInformers.Core().V1().Pods().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{AddFunc: hint, UpdateFunc: func(_, obj any) { hint(obj) }}) - s.podInformers.Start(ctx.Done()) + _, _ = s.pods.Core().V1().Pods().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { hint(nil, obj) }, + UpdateFunc: hint, + DeleteFunc: func(any) { hint(nil, nil) }, + }) } // Start is called after container discovery is running and before the generic @@ -248,7 +269,9 @@ func (s *Service) Start(ctx context.Context) error { } s.mu.Unlock() go s.exporter.Run(runCtx) - s.watchPods(runCtx) + if s.pods != nil { + s.pods.Start(runCtx.Done()) + } // Drain observed startup churn into the readiness barrier before handing // work to the background loop. Persistent churn stays explicitly NotReady. drain: @@ -297,17 +320,19 @@ func (s *Service) refresh(ctx context.Context) { if s.options.PrivilegeImageName != "" && s.options.Kubernetes != nil && s.Controller.HasPrivilegePolicies() && (s.podsScheduled.Swap(false) || time.Since(s.lastPrepare) >= prepareInterval) { s.lastPrepare = time.Now() - scheduled, qos, err := ScheduledContainers(ctx, s.options.Kubernetes, s.options.NodeName) + scheduled, qos, namespaceLabels, err := ScheduledContainers(ctx, s.options.Kubernetes, s.options.NodeName) if err == nil { - s.scheduled, s.qos, s.podPollUntil = scheduled, qos, s.lastPrepare.Add(prepareInterval) + s.scheduled, s.qos, s.namespaceLabels = scheduled, qos, namespaceLabels err = s.Controller.PrepareScheduled(ctx, scheduled) } if err != nil { s.report(fmt.Errorf("privilege slot preparation: %w", err)) } } - // Every refresh, so a policy change also reaches the pod bindings. + // Every refresh, so a policy change also reaches the pod bindings: before + // the container bindings change, and after, when they created a slot. s.bindPods(ctx) + defer s.bindPods(ctx) // Configuration and selected-runtime discovery share one commit barrier. A // candidate cannot report Ready between its relist and a failed binding list. err := s.Controller.Refresh(ctx, func(ctx context.Context, selected func(sandboxpolicy.Binding) bool) ([]sandboxpolicy.Binding, error) { @@ -340,59 +365,87 @@ func (s *Service) refresh(ctx context.Context) { } // bindPods binds the pod cgroup of every scheduled pod that has a pod plan -// (see Controller.PodPlans) and unbinds the others. It uses the pods of the -// last slot preparation and makes no API call. A pod whose cgroup does not -// exist yet sets podPolling, so it is checked again soon. +// (see Controller.PodPlans) and unbinds the others. It reads the node's pods +// from the informer cache and makes no API call. A pod whose cgroup does not +// exist yet sets podPolling, so it is checked again soon, for up to +// prepareInterval after it was first seen. func (s *Service) bindPods(ctx context.Context) { binder, ok := s.backend.(sandboxpolicy.PodBinder) if !ok || s.Controller == nil { return } - plans, ok := s.Controller.PodPlans(s.scheduled) + scheduled, qos := s.scheduled, s.qos + if s.pods != nil { + pods, err := s.pods.Core().V1().Pods().Lister().List(labels.Everything()) + if err != nil { + s.report(fmt.Errorf("privilege pod binding: %w", err)) + return + } + scheduled, qos = scheduledContainers(pods, s.namespaceLabels, s.options.NodeName) + } + plans, ok := s.Controller.PodPlans(scheduled) if !ok { return // the policies are not known: keep the pod bindings } var pods []sandboxpolicy.PodBinding - missing := false + now := time.Now() + firstSeen := map[types.UID]time.Time{} + polling := false if len(plans) > 0 { - layout, known := s.podCgroupLayout() - for _, c := range s.scheduled { + layout, known, containerCgroups := s.podCgroupLayout() + for _, c := range scheduled { plan, ok := plans[c.PodUID] if !ok { continue } delete(plans, c.PodUID) - path := layout.path(c.PodUID, s.qos[c.PodUID]) + seen, ok := s.podFirstSeen[c.PodUID] + if !ok { + seen = now + } + firstSeen[c.PodUID] = seen + path := layout.path(c.PodUID, qos[c.PodUID]) if !known || path == "" || s.podCgroupID == nil { continue } id, err := s.podCgroupID(path) if err != nil || id == 0 { - missing = true + polling = polling || now.Sub(seen) < prepareInterval + continue + } + // A runtime that runs the container in the pod cgroup itself: + // the container binding owns it. + if containerCgroups[id] { continue } pods = append(pods, sandboxpolicy.PodBinding{PodUID: c.PodUID, Namespace: c.Namespace, PodName: c.PodName, CgroupID: id, CgroupPath: path, Plan: plan}) } } - s.podPolling = missing && time.Now().Before(s.podPollUntil) + s.podFirstSeen, s.podPolling = firstSeen, polling if err := binder.BindPods(ctx, pods); err != nil { s.report(fmt.Errorf("privilege pod binding: %w", err)) } } -// podCgroupLayout reads the pod cgroup layout from a running pod container. -func (s *Service) podCgroupLayout() (podCgroupLayout, bool) { +// podCgroupLayout reads the pod cgroup layout from a running pod container, +// and returns the cgroup IDs of the running containers. +func (s *Service) podCgroupLayout() (podCgroupLayout, bool, map[uint64]bool) { var layout podCgroupLayout found := false + cgroups := map[uint64]bool{} if s.options.Containers == nil { - return layout, false + return layout, false, cgroups } s.options.Containers.ContainerRange(func(c *containercollection.Container) { - if !found && c != nil { + if c == nil { + return + } + cgroups[c.CgroupID] = true + if !found { layout, found = learnPodCgroupLayout(c.CgroupV2, types.UID(c.K8s.PodUID)) } }) - return layout, found + return layout, found, cgroups } // ContainerCallback is deliberately nonblocking: event delivery must never hold @@ -441,8 +494,8 @@ func (s *Service) Close(ctx context.Context) error { case <-ctx.Done(): return ctx.Err() } - if s.podInformers != nil { - s.podInformers.Shutdown() + if s.pods != nil { + s.pods.Shutdown() } } if !s.backendClosed { diff --git a/pkg/sandboxpolicy/integration/service_test.go b/pkg/sandboxpolicy/integration/service_test.go index 8539199cfb..9bba3e0f20 100644 --- a/pkg/sandboxpolicy/integration/service_test.go +++ b/pkg/sandboxpolicy/integration/service_test.go @@ -10,10 +10,12 @@ import ( containercollection "github.com/inspektor-gadget/inspektor-gadget/pkg/container-collection" "github.com/kubescape/node-agent/pkg/sandboxpolicy" api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" dynamicfake "k8s.io/client-go/dynamic/fake" ) @@ -265,3 +267,78 @@ func TestServiceDrainsStartupChurnBeforeReady(t *testing.T) { t.Fatalf("stable replay never became Ready: %+v", last.Status.Conditions) } } + +type podBinderBackend struct { + serviceTestBackend + pods [][]sandboxpolicy.PodBinding +} + +func (b *podBinderBackend) BindPods(_ context.Context, pods []sandboxpolicy.PodBinding) error { + b.pods = append(b.pods, pods) + return nil +} + +func TestServiceBindsPodCgroups(t *testing.T) { + client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{sandboxpolicy.PolicyGVR: "SandboxPolicyList", sandboxpolicy.ClusterPolicyGVR: "ClusterSandboxPolicyList", sandboxpolicy.ClassesGVR: "SandboxPolicyClassesList"}) + p := &unstructured.Unstructured{Object: map[string]any{"apiVersion": api.SchemeGroupVersion.String(), "kind": "SandboxPolicy", "metadata": map[string]any{"name": "priv", "namespace": "ns", "uid": "policy-uid", "generation": int64(1)}, "spec": map[string]any{"selector": map[string]any{"podSelector": map[string]any{}}, "mode": "enforce", "containers": []any{"*"}, "privilege": map[string]any{"default": "allow", "rules": []any{map[string]any{"name": "no-chroot", "action": "deny", "target": map[string]any{"syscall": []any{"chroot"}}}}}}}} + if _, err := client.Resource(sandboxpolicy.PolicyGVR).Namespace("ns").Create(t.Context(), p, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + backend := &podBinderBackend{} + controller, err := sandboxpolicy.NewController(client, backend, nil, nil, sandboxpolicy.Options{Node: api.NodeReference{Name: "node", UID: "node-uid"}, AgentNamespace: "agent"}) + if err != nil { + t.Fatal(err) + } + if err := controller.InitialSync(t.Context()); err != nil { + t.Fatal(err) + } + // A running container of another pod gives the layout. + containers := &containercollection.ContainerCollection{} + running := &containercollection.Container{CgroupID: 500, CgroupV2: "/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podother_uid.slice/cri-containerd-x.scope"} + running.Runtime.ContainerID = "x" + running.K8s.PodUID = "other-uid" + containers.AddContainer(running) + want := "/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podpod_uid.slice" + var id uint64 + s := &Service{Controller: controller, backend: backend, options: ServiceOptions{Containers: containers, NodeName: "node"}, + scheduled: []sandboxpolicy.Binding{{PodUID: "pod-uid", Namespace: "ns", PodName: "pod", ContainerName: "app", Kind: "regular"}}, + qos: map[types.UID]corev1.PodQOSClass{"pod-uid": corev1.PodQOSBestEffort}, + podCgroupID: func(path string) (uint64, error) { + if path != want || id == 0 { + return 0, errors.New("no such cgroup") + } + return id, nil + }} + last := func() []sandboxpolicy.PodBinding { return backend.pods[len(backend.pods)-1] } + + // The pod cgroup does not exist yet: nothing is bound, and the pod is polled. + s.bindPods(t.Context()) + if len(last()) != 0 || !s.podPolling { + t.Fatalf("missing cgroup: %+v polling=%v", last(), s.podPolling) + } + id = 77 + s.bindPods(t.Context()) + if got := last(); len(got) != 1 || got[0].CgroupID != 77 || got[0].CgroupPath != want || len(got[0].Plan.Policies) != 1 || s.podPolling { + t.Fatalf("bind: %+v polling=%v", got, s.podPolling) + } + // A container that runs in the pod cgroup itself keeps its own binding. + id = 500 + s.bindPods(t.Context()) + if len(last()) != 0 { + t.Fatalf("pod cgroup shared with a container must not be bound: %+v", last()) + } + // Polling stops when the pod was first seen too long ago. + id = 0 + s.podFirstSeen["pod-uid"] = time.Now().Add(-prepareInterval) + s.bindPods(t.Context()) + if s.podPolling { + t.Fatal("polling must stop after prepareInterval") + } + // An init container that the wildcard does not select: no pod binding. + id = 77 + s.scheduled = append(s.scheduled, sandboxpolicy.Binding{PodUID: "pod-uid", Namespace: "ns", PodName: "pod", ContainerName: "setup", Kind: "init"}) + s.bindPods(t.Context()) + if len(last()) != 0 { + t.Fatalf("mixed pod must not be bound: %+v", last()) + } +} From 1074f1deb44e9833f177290399044b9b5528610f Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 18:09:08 +0200 Subject: [PATCH 30/54] fix(sandboxpolicy): load the privilege gadget before the first container The first gadget load after a node-agent start takes about 3 s, so the first container of a privilege policy could start before any instance was ready and run its first commands unchecked. Prepare now starts the node instance (with no slot) when none runs; the service prepares as soon as a privilege policy exists. Later loads are fast, so the first pod's slot is ready in time. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/igbackend/privilege.go | 5 ++++- .../igbackend/privilege_kernel_test.go | 11 +++++++++-- pkg/sandboxpolicy/igbackend/privilege_test.go | 18 ++++++++++++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/pkg/sandboxpolicy/igbackend/privilege.go b/pkg/sandboxpolicy/igbackend/privilege.go index a03e3b8e24..91262df264 100644 --- a/pkg/sandboxpolicy/igbackend/privilege.go +++ b/pkg/sandboxpolicy/igbackend/privilege.go @@ -292,7 +292,10 @@ func (b *PrivilegeBackend) Prepare(ctx context.Context, plans []sandboxpolicy.Pl return fmt.Errorf("privilege backend is closed") } prepared := map[string][]sandboxpolicy.CompiledPolicy{} - missing := false + // Without an instance, start one even with no plans: the first gadget load + // is slow, so the first container of a privilege policy would find no + // instance ready (for example the first pod after a node-agent restart). + missing := b.instance == nil for _, plan := range plans { combination, policies := Combination(plan) if len(policies) == 0 { diff --git a/pkg/sandboxpolicy/igbackend/privilege_kernel_test.go b/pkg/sandboxpolicy/igbackend/privilege_kernel_test.go index 98e6a7076c..3659695cd7 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_kernel_test.go +++ b/pkg/sandboxpolicy/igbackend/privilege_kernel_test.go @@ -87,6 +87,13 @@ func TestPrivilegeBackendKernel(t *testing.T) { if err = unshare(); err != nil { t.Fatalf("baseline unshare failed: %v", err) } + // An instance with no slot loads before any container needs it. + if err := backend.Prepare(ctx, nil); err != nil || backend.Stats().Replacements != 1 { + t.Fatalf("empty instance: %v %+v", err, backend.Stats()) + } + if err = unshare(); err != nil { + t.Fatalf("an empty instance must judge nothing: %v", err) + } if r, err := backend.Replace(ctx, binding, plan("deny", api.ActionDeny)); err != nil || !r.Committed { t.Fatalf("deny plan: %+v %v", r, err) } @@ -101,7 +108,7 @@ func TestPrivilegeBackendKernel(t *testing.T) { if r, err := backend.Replace(ctx, second, plan("deny", api.ActionDeny)); err != nil || !r.Committed { t.Fatalf("second container, same policy: %+v %v", r, err) } - if s := backend.Stats(); s.Prepared != 1 || s.Replacements != 1 { + if s := backend.Stats(); s.Prepared != 1 || s.Replacements != 2 { t.Fatalf("same policy must be a live bind: %+v", s) } if unshareIn(id2) == nil { @@ -112,7 +119,7 @@ func TestPrivilegeBackendKernel(t *testing.T) { if r, err := backend.Replace(ctx, second, other); err != nil || !r.Committed { t.Fatalf("second container, other policy: %+v %v", r, err) } - if s := backend.Stats(); s.Replacements != 2 { + if s := backend.Stats(); s.Replacements != 3 { t.Fatalf("a new combination must replace the instance: %+v", s) } if unshareIn(id2) != nil { diff --git a/pkg/sandboxpolicy/igbackend/privilege_test.go b/pkg/sandboxpolicy/igbackend/privilege_test.go index 9c84662219..469c52585a 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_test.go +++ b/pkg/sandboxpolicy/igbackend/privilege_test.go @@ -362,3 +362,21 @@ func TestPrivilegePodUnboundWhenNewSlotIsMissing(t *testing.T) { t.Fatal("the pod must be bound once the new slot exists") } } + +func TestPrivilegePrepareStartsInstanceWithoutPlans(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + // A privilege policy exists but no pod uses it yet: load the gadget now, + // so the first container does not wait for the slow first load. + if err := b.Prepare(ctx, nil); err != nil || len(e.instances) != 1 || e.configs[0].Policy.Slots != 0 { + t.Fatalf("first prepare: %v instances=%d", err, len(e.instances)) + } + if err := b.Prepare(ctx, nil); err != nil || len(e.instances) != 1 { + t.Fatalf("a running instance must be kept: %v instances=%d", err, len(e.instances)) + } + plan := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + if _, err := b.Replace(ctx, privBinding(1), plan); err != nil || len(e.instances) != 2 { + t.Fatalf("first container: %v instances=%d", err, len(e.instances)) + } +} From ab230ec62c2fe2a8566684af9f9dc6cbc2b519b1 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 18:28:23 +0200 Subject: [PATCH 31/54] fix(sandboxpolicy): address review findings on pod bindings - A pod binding needs the same checks as the container binding: an invalid policy, a class error, too many policies or a policy that needs a ContainerProfile give no pod plan. - Read namespace labels from a namespace informer, and the node's pods from the pod informer, also for slot preparation (no List per preparation). - Stop polling when the pod plans are not known. - Retry the replacement without pod bindings only when they add slots. - Wait 30 s before another start of an instance with no slot. - Document what a parent binding also judges. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/enforce_privilege/README.md | 7 ++ pkg/sandboxpolicy/controller.go | 58 ++++++++++++++-- pkg/sandboxpolicy/igbackend/privilege.go | 35 +++++++++- pkg/sandboxpolicy/igbackend/privilege_test.go | 17 +++++ pkg/sandboxpolicy/integration/bindings.go | 14 ++-- pkg/sandboxpolicy/integration/service.go | 69 +++++++++++++++---- pkg/sandboxpolicy/sections_test.go | 25 +++++++ 7 files changed, 195 insertions(+), 30 deletions(-) diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index ebbadf6d96..6878aef032 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -125,6 +125,13 @@ policy: ## Known limits +- **Parent bindings.** A bound parent cgroup also judges the armed tasks of + its child cgroups that have no binding of their own: OCI `createContainer` + and `startContainer` hooks (runc runs them in the container mount + namespace before the workload), and an infrastructure (pause) process. A + new instance does not arm the tasks that run only under a parent binding; + they are judged again when their own cgroup is bound or at their next exec. + - **Lost mark at exec.** The exec hook cannot fail the exec, so a task whose mark fails there stays unarmed until its next exec. In an enforcing slot this is a tracking fault (see above). A later check cannot tell such a task apart diff --git a/pkg/sandboxpolicy/controller.go b/pkg/sandboxpolicy/controller.go index c91f39063b..11128db1c9 100644 --- a/pkg/sandboxpolicy/controller.go +++ b/pkg/sandboxpolicy/controller.go @@ -52,6 +52,9 @@ type Controller struct { lastError error bindingError error configError error + // classes and classErr are from the last reconcile (see PodPlans). + classes api.PolicyClasses + classErr error } var _ watcher.Adaptor = (*Controller)(nil) @@ -197,23 +200,32 @@ func (c *Controller) PrepareScheduled(ctx context.Context, containers []Binding) // Such a pod can be bound at its pod cgroup before its containers start. A pod // whose containers get different policies, or none, has no pod plan: a pod // binding would also judge the containers that the policies do not select. -// It returns false when the policies are not known (a configuration error). +// A container must also pass the checks that its own binding needs (see +// podContainerPlan), so a pod binding never applies a policy that the +// container binding refuses. It returns false when the policies are not known +// (a configuration error). func (c *Controller) PodPlans(containers []Binding) (map[types.UID]Plan, bool) { c.mu.Lock() if c.configError != nil { c.mu.Unlock() return nil, false } - plans := privilegePlans(c.privilegePolicies(), containers) + var policies []desiredPolicy + for _, obj := range c.objects { + if obj.GetKind() != "SandboxPolicyClasses" { + policies = append(policies, parsePolicy(obj)) + } + } + classes, classErr := c.classes, c.classErr c.mu.Unlock() pods := map[types.UID]Plan{} mixed := map[types.UID]bool{} - for i, b := range containers { - plan := plans[i] + for _, b := range containers { if mixed[b.PodUID] { continue } - if len(plan.Policies) == 0 { + plan, ok := podContainerPlan(policies, classes, classErr, b) + if !ok { mixed[b.PodUID] = true delete(pods, b.PodUID) continue @@ -229,6 +241,41 @@ func (c *Controller) PodPlans(containers []Binding) (map[types.UID]Plan, bool) { return pods, true } +// podContainerPlan returns the privilege plan of a container for a pod +// binding. It is false when the container's own binding would fail, as in +// reconcile: an invalid policy that may select it, a class error, too many +// policies, or a policy that does not compile without a ContainerProfile +// resolver (a policy with sources depends on the profile, so it is refused). +func podContainerPlan(policies []desiredPolicy, classes api.PolicyClasses, classErr error, b Binding) (Plan, bool) { + var plan Plan + selected := 0 + for _, p := range policies { + if p.err != nil { + if p.identity.Kind == "SandboxPolicy" && p.identity.Namespace != b.Namespace { + continue + } + return Plan{}, false + } + if p.spec.Mode == api.ModeOff || !p.matches(b) { + continue + } + selected++ + if classErr != nil { + return Plan{}, false + } + if _, err := Compile(context.Background(), p.identity, p.spec, classes, b, nil); err != nil { + return Plan{}, false + } + if p.spec.Privilege != nil { + plan.Policies = append(plan.Policies, CompiledPolicy{Identity: p.identity, Mode: p.spec.Mode, Privilege: p.spec.Privilege.DeepCopy()}) + } + } + if selected > MaxPoliciesPerBinding || len(plan.Policies) == 0 { + return Plan{}, false + } + return plan, true +} + // privilegePlans returns the plan of each container, in order (possibly empty). func privilegePlans(policies []desiredPolicy, containers []Binding) []Plan { plans := make([]Plan, len(containers)) @@ -414,6 +461,7 @@ func (c *Controller) reconcile(ctx context.Context) error { classes.PathClasses[name] = paths } } + c.classes, c.classErr = classes, classErr policyErrors := make(map[string]error) for _, p := range policies { if p.err != nil { diff --git a/pkg/sandboxpolicy/igbackend/privilege.go b/pkg/sandboxpolicy/igbackend/privilege.go index 91262df264..40016a3867 100644 --- a/pkg/sandboxpolicy/igbackend/privilege.go +++ b/pkg/sandboxpolicy/igbackend/privilege.go @@ -9,6 +9,7 @@ import ( "fmt" "sort" "sync" + "time" "github.com/kubescape/go-logger" "github.com/kubescape/go-logger/helpers" @@ -110,8 +111,14 @@ type PrivilegeBackend struct { retiring []PrivilegeInstance stats PrivilegeStats closed bool + // emptyFailed is when starting an instance with no slot last failed. + emptyFailed time.Time } +// emptyRetry is the shortest time between two starts of an instance with no +// slot: each start is a slow gadget load on the refresh loop. +const emptyRetry = 30 * time.Second + var ( _ sandboxpolicy.Backend = (*PrivilegeBackend)(nil) _ sandboxpolicy.ProtectionAttester = (*PrivilegeBackend)(nil) @@ -225,7 +232,7 @@ func (b *PrivilegeBackend) Replace(ctx context.Context, binding sandboxpolicy.Bi } extra := map[string][]sandboxpolicy.CompiledPolicy{combination: policies} err := b.replaceInstance(ctx, b.prepared, extra, k, next, true) - if err != nil && (len(b.prepared) > 0 || len(b.pods) > 0) { + if err != nil && (len(b.prepared) > 0 || b.podsAddSlots(extra)) { // The prepared set and the pod bindings may not fit; never let them // block a real container. BindPods binds the pods again later. err = b.replaceInstance(ctx, nil, extra, k, next, false) @@ -295,7 +302,7 @@ func (b *PrivilegeBackend) Prepare(ctx context.Context, plans []sandboxpolicy.Pl // Without an instance, start one even with no plans: the first gadget load // is slow, so the first container of a privilege policy would find no // instance ready (for example the first pod after a node-agent restart). - missing := b.instance == nil + missing := b.instance == nil && time.Since(b.emptyFailed) >= emptyRetry for _, plan := range plans { combination, policies := Combination(plan) if len(policies) == 0 { @@ -311,7 +318,11 @@ func (b *PrivilegeBackend) Prepare(ctx context.Context, plans []sandboxpolicy.Pl return b.retire(ctx) } // The prepared set is kept only when the instance with it is running. - return errors.Join(b.replaceInstance(ctx, prepared, nil, identity{}, nil, true), b.retire(ctx)) + err := b.replaceInstance(ctx, prepared, nil, identity{}, nil, true) + if err != nil && len(prepared) == 0 { + b.emptyFailed = time.Now() + } + return errors.Join(err, b.retire(ctx)) } // BindPods binds the given pod cgroups to the slot of their plan and unbinds @@ -370,6 +381,24 @@ func (b *PrivilegeBackend) BindPods(ctx context.Context, pods []sandboxpolicy.Po return errors.Join(append(errs, b.retire(ctx))...) } +// podsAddSlots reports whether a pod binding needs a combination that no +// container binding, and not extra, needs. +func (b *PrivilegeBackend) podsAddSlots(extra map[string][]sandboxpolicy.CompiledPolicy) bool { + used := map[string]bool{} + for c := range extra { + used[c] = true + } + for _, v := range b.bindings { + used[v.combination] = true + } + for _, p := range b.pods { + if !used[p.combination] { + return true + } + } + return false +} + // replaceInstance starts a candidate with every combination in use or prepared // (plus extra), binds every container (plus next, under nextKey) and, with // withPods, every pod cgroup, and only then stops the old instance. Without diff --git a/pkg/sandboxpolicy/igbackend/privilege_test.go b/pkg/sandboxpolicy/igbackend/privilege_test.go index 469c52585a..b6ce5655de 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_test.go +++ b/pkg/sandboxpolicy/igbackend/privilege_test.go @@ -380,3 +380,20 @@ func TestPrivilegePrepareStartsInstanceWithoutPlans(t *testing.T) { t.Fatalf("first container: %v instances=%d", err, len(e.instances)) } } + +func TestPrivilegeEmptyInstanceRetryWaits(t *testing.T) { + e := &fakePrivilegeEngine{startErr: errors.New("load failed")} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + if err := b.Prepare(ctx, nil); err == nil { + t.Fatal("the failed start must be reported") + } + e.startErr = nil + if err := b.Prepare(ctx, nil); err != nil || len(e.instances) != 0 { + t.Fatalf("an empty start must not be retried at once: %v instances=%d", err, len(e.instances)) + } + // A real plan is not delayed. + if err := b.Prepare(ctx, []sandboxpolicy.Plan{privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount))}); err != nil || len(e.instances) != 1 { + t.Fatalf("prepare with a plan: %v instances=%d", err, len(e.instances)) + } +} diff --git a/pkg/sandboxpolicy/integration/bindings.go b/pkg/sandboxpolicy/integration/bindings.go index 92925fe959..52bc67ebb2 100644 --- a/pkg/sandboxpolicy/integration/bindings.go +++ b/pkg/sandboxpolicy/integration/bindings.go @@ -64,20 +64,20 @@ func SnapshotSelectedBindingsFromContainers(ctx context.Context, client kubernet // ScheduledContainers returns API metadata (no runtime identity) for every // declared regular, init and ephemeral container of the pods scheduled to this -// node that are not finished, the QoS class of each such pod (see -// scheduledContainers) and the namespace labels. It feeds privilege slot +// node that are not finished, and the QoS class of each such pod (see +// scheduledContainers). It feeds privilege slot // preparation and pod cgroup binding, not container binding. -func ScheduledContainers(ctx context.Context, client kubernetes.Interface, nodeName string) ([]sandboxpolicy.Binding, map[types.UID]corev1.PodQOSClass, map[string]map[string]string, error) { +func ScheduledContainers(ctx context.Context, client kubernetes.Interface, nodeName string) ([]sandboxpolicy.Binding, map[types.UID]corev1.PodQOSClass, error) { if client == nil || nodeName == "" { - return nil, nil, nil, fmt.Errorf("Kubernetes client and node name are required") + return nil, nil, fmt.Errorf("Kubernetes client and node name are required") } pods, err := client.CoreV1().Pods("").List(ctx, metav1.ListOptions{FieldSelector: fields.OneTermEqualSelector("spec.nodeName", nodeName).String()}) if err != nil { - return nil, nil, nil, fmt.Errorf("list node pods: %w", err) + return nil, nil, fmt.Errorf("list node pods: %w", err) } namespaces, err := client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) if err != nil { - return nil, nil, nil, fmt.Errorf("list namespaces: %w", err) + return nil, nil, fmt.Errorf("list namespaces: %w", err) } namespaceLabels := make(map[string]map[string]string, len(namespaces.Items)) for _, ns := range namespaces.Items { @@ -88,7 +88,7 @@ func ScheduledContainers(ctx context.Context, client kubernetes.Interface, nodeN list[i] = &pods.Items[i] } result, qos := scheduledContainers(list, namespaceLabels, nodeName) - return result, qos, namespaceLabels, nil + return result, qos, nil } // scheduledContainers is ScheduledContainers for a given pod list. A pod of a diff --git a/pkg/sandboxpolicy/integration/service.go b/pkg/sandboxpolicy/integration/service.go index ce7dba7bdc..152c39e527 100644 --- a/pkg/sandboxpolicy/integration/service.go +++ b/pkg/sandboxpolicy/integration/service.go @@ -83,20 +83,19 @@ type Service struct { podsScheduled atomic.Bool lastPrepare time.Time // The fields below are used only by the refresh loop. - // namespaceLabels are from the last slot preparation. scheduled and qos - // are the containers and QoS classes of the node's pods from the last - // preparation, used when there is no pod informer. - namespaceLabels map[string]map[string]string - scheduled []sandboxpolicy.Binding - qos map[types.UID]corev1.PodQOSClass + // scheduled and qos are the containers and QoS classes of the node's pods + // from the last slot preparation, used when there are no informers. + scheduled []sandboxpolicy.Binding + qos map[types.UID]corev1.PodQOSClass // podFirstSeen is when a pod with a pod plan was first seen. podPolling is // set while such a pod has no pod cgroup yet, for up to prepareInterval. podFirstSeen map[types.UID]time.Time podPolling bool // podCgroupID returns the cgroup ID of a pod cgroup path (replaced in tests). podCgroupID func(string) (uint64, error) - // pods is the node's pod informer (nil without a privilege image). - pods informers.SharedInformerFactory + // pods and namespaces are the informers of the node's pods and of all + // namespaces (nil without a privilege image). + pods, namespaces informers.SharedInformerFactory } // prepareInterval is the longest time between two slot preparations. @@ -133,6 +132,7 @@ func NewService(ctx context.Context, options ServiceOptions) (*Service, error) { s.pods = informers.NewSharedInformerFactoryWithOptions(options.Kubernetes, 0, informers.WithTweakListOptions(func(o *metav1.ListOptions) { o.FieldSelector = fields.OneTermEqualSelector("spec.nodeName", options.NodeName).String() })) + s.namespaces = informers.NewSharedInformerFactory(options.Kubernetes, 0) s.watchPods() } s.exporter, err = sandboxpolicy.NewVerdictExporter(sandboxpolicy.ExporterOptions{Capacity: 1024, Receivers: options.Receivers, Enricher: options.Enricher, ResolveBinding: s.resolveBinding}) @@ -242,6 +242,15 @@ func (s *Service) watchPods() { s.ContainerCallback(containercollection.PubSubEvent{}) } } + _, _ = s.namespaces.Core().V1().Namespaces().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + UpdateFunc: func(old, obj any) { + before, ok1 := old.(*corev1.Namespace) + after, ok2 := obj.(*corev1.Namespace) + if !ok1 || !ok2 || !stdmaps.Equal(before.Labels, after.Labels) { + s.ContainerCallback(containercollection.PubSubEvent{}) + } + }, + }) _, _ = s.pods.Core().V1().Pods().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj any) { hint(nil, obj) }, UpdateFunc: hint, @@ -271,6 +280,12 @@ func (s *Service) Start(ctx context.Context) error { go s.exporter.Run(runCtx) if s.pods != nil { s.pods.Start(runCtx.Done()) + s.namespaces.Start(runCtx.Done()) + // Bounded: the caches only add hints, refresh works without them. + syncCtx, cancelSync := context.WithTimeout(runCtx, 10*time.Second) + s.pods.WaitForCacheSync(syncCtx.Done()) + s.namespaces.WaitForCacheSync(syncCtx.Done()) + cancelSync() } // Drain observed startup churn into the readiness barrier before handing // work to the background loop. Persistent churn stays explicitly NotReady. @@ -315,14 +330,14 @@ drain: } func (s *Service) refresh(ctx context.Context) { s.snapshotEpoch.Store(s.inputEpoch.Load()) - // Prepare after a Pending-pod event, else at most every prepareInterval, - // so the extra pod and namespace listing does not run on every refresh. + // Prepare after a Pending-pod or policy event, else at most every + // prepareInterval. if s.options.PrivilegeImageName != "" && s.options.Kubernetes != nil && s.Controller.HasPrivilegePolicies() && (s.podsScheduled.Swap(false) || time.Since(s.lastPrepare) >= prepareInterval) { s.lastPrepare = time.Now() - scheduled, qos, namespaceLabels, err := ScheduledContainers(ctx, s.options.Kubernetes, s.options.NodeName) + scheduled, qos, err := s.scheduledPods(ctx) if err == nil { - s.scheduled, s.qos, s.namespaceLabels = scheduled, qos, namespaceLabels + s.scheduled, s.qos = scheduled, qos err = s.Controller.PrepareScheduled(ctx, scheduled) } if err != nil { @@ -374,14 +389,14 @@ func (s *Service) bindPods(ctx context.Context) { if !ok || s.Controller == nil { return } + s.podPolling = false // set again below when a pod cgroup is missing scheduled, qos := s.scheduled, s.qos if s.pods != nil { - pods, err := s.pods.Core().V1().Pods().Lister().List(labels.Everything()) - if err != nil { + var err error + if scheduled, qos, err = s.scheduledPods(ctx); err != nil { s.report(fmt.Errorf("privilege pod binding: %w", err)) return } - scheduled, qos = scheduledContainers(pods, s.namespaceLabels, s.options.NodeName) } plans, ok := s.Controller.PodPlans(scheduled) if !ok { @@ -427,6 +442,29 @@ func (s *Service) bindPods(ctx context.Context) { } } +// scheduledPods returns the containers and QoS classes of the node's pods (see +// ScheduledContainers), from the informer caches when there are informers. +func (s *Service) scheduledPods(ctx context.Context) ([]sandboxpolicy.Binding, map[types.UID]corev1.PodQOSClass, error) { + if s.pods == nil { + scheduled, qos, err := ScheduledContainers(ctx, s.options.Kubernetes, s.options.NodeName) + return scheduled, qos, err + } + pods, err := s.pods.Core().V1().Pods().Lister().List(labels.Everything()) + if err != nil { + return nil, nil, err + } + namespaces, err := s.namespaces.Core().V1().Namespaces().Lister().List(labels.Everything()) + if err != nil { + return nil, nil, err + } + namespaceLabels := make(map[string]map[string]string, len(namespaces)) + for _, ns := range namespaces { + namespaceLabels[ns.Name] = ns.Labels + } + scheduled, qos := scheduledContainers(pods, namespaceLabels, s.options.NodeName) + return scheduled, qos, nil +} + // podCgroupLayout reads the pod cgroup layout from a running pod container, // and returns the cgroup IDs of the running containers. func (s *Service) podCgroupLayout() (podCgroupLayout, bool, map[uint64]bool) { @@ -496,6 +534,7 @@ func (s *Service) Close(ctx context.Context) error { } if s.pods != nil { s.pods.Shutdown() + s.namespaces.Shutdown() } } if !s.backendClosed { diff --git a/pkg/sandboxpolicy/sections_test.go b/pkg/sandboxpolicy/sections_test.go index 071acf5e82..6ff9a75b2c 100644 --- a/pkg/sandboxpolicy/sections_test.go +++ b/pkg/sandboxpolicy/sections_test.go @@ -3,6 +3,7 @@ package sandboxpolicy import ( "context" "errors" + "strings" "testing" api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" @@ -330,3 +331,27 @@ func TestControllerPodPlans(t *testing.T) { t.Fatalf("only the pod whose containers all get the same policies has a pod plan: %+v", plans) } } + +func TestControllerPodPlansNeedContainerEligibility(t *testing.T) { + ctx := context.Background() + priv := policy("priv", "ns") + priv.Spec.Privilege = privilegeSection(api.ActionDeny) + // A second policy that selects the same container but depends on a + // ContainerProfile: the container binding can fail, so no pod binding. + sourced := policy("sourced", "ns") + sourced.Spec.Selector.PodSelector = metav1.LabelSelector{MatchLabels: map[string]string{"profile": "yes"}} + sourced.Spec.Sources = []api.PolicySource{{ContainerProfile: api.ContainerProfileReference{Name: "profile"}, Include: []string{"execs"}}} + c, _, _, _ := setup(t, priv, sourced) + if err := c.InitialSync(ctx); err != nil && !strings.Contains(err.Error(), "ContainerProfile") { + t.Fatal(err) + } + plain := binding() + plain.ContainerID, plain.CgroupID, plain.PodUID = "", 0, "plain" + profiled := plain + profiled.PodUID = "profiled" + profiled.PodLabels = map[string]string{"app": "test", "profile": "yes"} + plans, ok := c.PodPlans([]Binding{plain, profiled}) + if !ok || len(plans) != 1 || len(plans["plain"].Policies) != 1 { + t.Fatalf("only the pod without a sourced policy has a pod plan: %+v", plans) + } +} From 5340fd6e8c4371440fc0ad489446e4fed68ee660 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:09:23 +0200 Subject: [PATCH 32/54] fix(enforcement): arm the group leader of a thread that cannot be opened On kernels without PIDFD_THREAD a non-leader thread has no pidfd, and the BPF rule judges it through its group leader. The walk now arms that leader, which can be outside the walked cgroup. A leader in the runtime mount namespace is not armed; the walk then returns an error. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- .../gadgets/common/enforcement/arm_linux.go | 43 +++++++++++++++++-- .../common/enforcement/arm_linux_test.go | 42 ++++++++++++++++++ 2 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 pkg/ebpf/gadgets/common/enforcement/arm_linux_test.go diff --git a/pkg/ebpf/gadgets/common/enforcement/arm_linux.go b/pkg/ebpf/gadgets/common/enforcement/arm_linux.go index 699fb6d4ed..bf7ac52d57 100644 --- a/pkg/ebpf/gadgets/common/enforcement/arm_linux.go +++ b/pkg/ebpf/gadgets/common/enforcement/arm_linux.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "golang.org/x/sys/unix" @@ -105,10 +106,14 @@ func armThreads(armed Map, threads string, hostMntns uint64, seen map[int]bool) if errors.Is(err, unix.EINVAL) { fd, err = unix.PidfdOpen(tid, 0) } - if errors.Is(err, unix.ESRCH) || errors.Is(err, unix.EINVAL) { - // Exited, or a non-leader thread on a kernel without - // PIDFD_THREAD: the BPF rule judges it through its leader. - seen[tid] = true + if errors.Is(err, unix.EINVAL) { + // A non-leader thread on a kernel without PIDFD_THREAD: the + // BPF rule judges it through its group leader, which can be + // outside this cgroup, so arm the leader. + fd, err = openLeader(scanner.Text(), hostMntns) + } + if errors.Is(err, unix.ESRCH) || errors.Is(err, fs.ErrNotExist) { + seen[tid] = true // exited continue } if err != nil { @@ -124,6 +129,36 @@ func armThreads(armed Map, threads string, hostMntns uint64, seen map[int]bool) return scanner.Err() } +// openLeader opens a pidfd of the thread group leader of tid. A leader in the +// runtime mount namespace is a runtime helper: arming it would judge the +// runtime, so the thread cannot be armed and the caller gets an error. +func openLeader(tid string, hostMntns uint64) (int, error) { + data, err := os.ReadFile(filepath.Join(host.HostProcFs, tid, "status")) + if err != nil { + return -1, err + } + tgid := -1 + for _, line := range strings.Split(string(data), "\n") { + if v, ok := strings.CutPrefix(line, "Tgid:"); ok { + tgid, err = strconv.Atoi(strings.TrimSpace(v)) + if err != nil { + return -1, fmt.Errorf("reading leader of task %s: %w", tid, err) + } + } + } + if tgid <= 0 { + return -1, fmt.Errorf("no leader for task %s", tid) + } + mntns, err := mntnsOf(strconv.Itoa(tgid)) + if err != nil { + return -1, err + } + if mntns == hostMntns { + return -1, fmt.Errorf("thread %s has a runtime helper as leader", tid) + } + return unix.PidfdOpen(tgid, 0) +} + // RuntimeMountNamespace is the mount namespace of PID 1 as this process sees // it: the namespace the container runtime runs in. Pass it to the gadget's // host_mntns_id so the BPF arming rule and ArmExisting agree. diff --git a/pkg/ebpf/gadgets/common/enforcement/arm_linux_test.go b/pkg/ebpf/gadgets/common/enforcement/arm_linux_test.go new file mode 100644 index 0000000000..8e5b78c27f --- /dev/null +++ b/pkg/ebpf/gadgets/common/enforcement/arm_linux_test.go @@ -0,0 +1,42 @@ +// Copyright 2026 The Kubescape Authors +// SPDX-License-Identifier: Apache-2.0 + +package enforcement + +import ( + "os" + "strconv" + "testing" + + "golang.org/x/sys/unix" +) + +func TestOpenLeaderOfThread(t *testing.T) { + entries, err := os.ReadDir("/proc/self/task") + if err != nil { + t.Skip("no /proc") + } + pid := strconv.Itoa(os.Getpid()) + thread := "" + for _, e := range entries { + if e.Name() != pid { + thread = e.Name() + } + } + if thread == "" { + t.Skip("no second thread") + } + fd, err := openLeader(thread, 0) + if err != nil { + t.Fatalf("leader of thread %s: %v", thread, err) + } + unix.Close(fd) + own, err := mntnsOf(pid) + if err != nil { + t.Fatal(err) + } + if fd, err := openLeader(thread, own); err == nil { + unix.Close(fd) + t.Fatal("a leader in the runtime mount namespace must not be armed") + } +} From ede3e9e3388f6b01ccd80fabd8c665247212fe83 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:09:23 +0200 Subject: [PATCH 33/54] fix(enforce_privilege): a slot enforces only when one of its entries enforces The META mode was enforce for any document in enforce mode, so a slot with only allow entries refused a fork whose child mark failed. It is now enforce only when an entry has the enforce deny rank, as the README says. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- .../gadgets/enforce_privilege/policy/kernel.go | 14 +++++--------- .../enforce_privilege/policy/kernel_test.go | 7 ++++++- .../runtime/runtime_kernel_test.go | 4 +++- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go index a292a917a4..2cda0ae362 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel.go @@ -178,15 +178,10 @@ func compileSlot(slot uint32, docs []Document, generation uint64, c *Compiled, o if err := d.Validate(); err != nil { return nil, err } - switch d.Policy.Mode { - case "enforce": - meta.Value.Mode = KernelModeEnforce - case "audit": - if meta.Value.Mode == KernelModeOff { - meta.Value.Mode = KernelModeAudit - } - } + // The META mode is audit for any active document and enforce only + // when an entry enforces (set below). if d.Policy.Mode != "off" { + meta.Value.Mode = KernelModeAudit active = append(active, d) } } @@ -313,7 +308,8 @@ func compileSlot(slot uint32, docs []Document, generation uint64, c *Compiled, o for k, v := range folded { keys = append(keys, k) // The META mode tells the program whether the slot enforces at - // all: an audit policy can carry an enforce rule. + // all: an audit policy can carry an enforce rule, and an enforce + // policy with only allow entries enforces nothing. if v.Rank == RankDenyEnforce { meta.Value.Mode = KernelModeEnforce } diff --git a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go index c9fc7da67b..d3766d5efb 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/policy/kernel_test.go @@ -169,9 +169,14 @@ func TestCompileSlotsMergesPolicies(t *testing.T) { if v := get(KernelKey{Slot: 1, Kind: KindDefault, ID: KindSyscall}); v.Rank != RankAllow { t.Fatalf("slot 1 default rank %d", v.Rank) } - if v := get(KernelKey{Slot: 1, Kind: KindMeta}); v.Generation != 4 || v.Mode != KernelModeEnforce { + // b enforces, but has only allow entries: the slot enforces nothing. + if v := get(KernelKey{Slot: 1, Kind: KindMeta}); v.Generation != 4 || v.Mode != KernelModeAudit { t.Fatalf("slot 1 meta %+v", v) } + // Slot 0 has a's enforce rule: it enforces. + if v := get(KernelKey{Slot: 0, Kind: KindMeta}); v.Mode != KernelModeEnforce { + t.Fatalf("slot 0 meta %+v", v) + } } // Policy A: audit deny mount. Policy B: enforce default deny, no mount rule. diff --git a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go index c35adfb140..897354ef64 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go +++ b/pkg/ebpf/gadgets/enforce_privilege/runtime/runtime_kernel_test.go @@ -171,7 +171,9 @@ func TestForkMarkFaultInjection(t *testing.T) { for _, mode := range []string{"enforce", "audit"} { t.Run(mode, func(t *testing.T) { f := newKernelFixture(t) - doc := privpolicy.Document{Version: 1, Name: "fault-" + mode, Policy: privpolicy.Policy{Mode: mode, Privilege: privpolicy.Section{Default: "allow"}}} + // One deny rule, so the slot enforces in enforce mode. + rule := privpolicy.Rule{Name: "no-modules", Action: "deny", Target: privpolicy.Target{Syscall: []string{"init_module"}}} + doc := privpolicy.Document{Version: 1, Name: "fault-" + mode, Policy: privpolicy.Policy{Mode: mode, Privilege: privpolicy.Section{Default: "allow", Rules: []privpolicy.Rule{rule}}}} compiled, err := privpolicy.CompileKernel(doc, 1) if err != nil { t.Fatal(err) From 43b2c9cbfb06f41460ce2a63fbef37a8fccb687b Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:09:23 +0200 Subject: [PATCH 34/54] docs(enforce_privilege): compat mknod and leaving the bound subtree For a 32-bit task every inode_mknod call is judged as mknod, including a unix socket bind. A task that can write to a cgroup outside its bound subtree runs there without the policy. Drop the verdict export item: the gadget has a verdict datasource. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/ebpf/gadgets/enforce_privilege/README.md | 12 +++++++++--- pkg/ebpf/gadgets/enforce_privilege/program.bpf.c | 4 +++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index 6878aef032..f198dced51 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -120,7 +120,9 @@ policy: `setns`, `open_tree`, `fsopen`, `fsmount`, `fspick`, `mount_setattr`) are judged at their `CAP_SYS_ADMIN` check by syscall number. For a 32-bit (compat) task the number is not mapped, so these checks fail closed: every - candidate syscall rule applies. + candidate syscall rule applies. The same holds for `mknod`: for a compat + task every `inode_mknod` call is judged as `mknod`, including a unix socket + `bind(2)`, so a `mknod` deny or a default deny also refuses that bind. - **Deny** returns `-EPERM` from the LSM hook, before the side effect. ## Known limits @@ -140,8 +142,12 @@ policy: - **Fault injection.** The `arming-fail-fork-marks` parameter makes every child mark fail at fork. It exists for tests (`runtime/runtime_kernel_test.go`); owners do not set it. -- **Not in this gadget yet:** export of verdicts as events (the owner logs - them), and kernels older than 6.1. +- **Kernels older than 6.1** are not supported yet. +- **Leaving the bound cgroup subtree.** A task that can write to a cgroup + outside its bound subtree (its `cgroup.procs`, or `clone3` with + `CLONE_INTO_CGROUP`) runs there without the policy. This needs write access + to the host cgroup tree, which a normal container does not have. A + `CLONE_INTO_CGROUP` clone is refused only when the child's mark fails. - **No-audit capability checks** (`CAP_OPT_NOAUDIT`) are excluded from capability enforcement. Most are kernel probes (for example the overcommit check), and judging them would deny normal work in every bound container. Some diff --git a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c index c7f894dc3c..5c5dd642ad 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c +++ b/pkg/ebpf/gadgets/enforce_privilege/program.bpf.c @@ -905,7 +905,9 @@ int BPF_PROG(ig_priv_key_permission) // vfs_mknod also runs for bind(2) of a unix socket path and for overlayfs // whiteouts (unlink or rename of a lower file). Judge only mknod(2) and -// mknodat(2). +// mknodat(2). A 32-bit (compat) task's syscall number is not mapped, so for +// it every call is judged as mknod (fail closed), its unix socket bind(2) +// included. SEC("lsm/inode_mknod") int BPF_PROG(ig_priv_mknod) { From 75a8f84407bb186017af9637e5ff4fb19ba713cd Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 20:14:02 +0200 Subject: [PATCH 35/54] fix(sandboxpolicy): heal tracking faults at most every 30 s Faults recur under memory pressure, and each same-plan Replace started a new node instance. A new instance only to heal faults now starts at most every healRetry; the binding stays unconfirmed meanwhile. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/igbackend/privilege.go | 15 +++++++++++++++ pkg/sandboxpolicy/igbackend/privilege_test.go | 9 +++++++++ 2 files changed, 24 insertions(+) diff --git a/pkg/sandboxpolicy/igbackend/privilege.go b/pkg/sandboxpolicy/igbackend/privilege.go index 40016a3867..556dc5660c 100644 --- a/pkg/sandboxpolicy/igbackend/privilege.go +++ b/pkg/sandboxpolicy/igbackend/privilege.go @@ -113,12 +113,19 @@ type PrivilegeBackend struct { closed bool // emptyFailed is when starting an instance with no slot last failed. emptyFailed time.Time + // lastHeal is when a tracking fault last started a new instance. + lastHeal time.Time } // emptyRetry is the shortest time between two starts of an instance with no // slot: each start is a slow gadget load on the refresh loop. const emptyRetry = 30 * time.Second +// healRetry is the shortest time between two new instances started only to +// heal tracking faults. Faults recur under memory pressure; a gadget reload on +// every reconcile would add load then. The binding stays unconfirmed meanwhile. +const healRetry = 30 * time.Second + var ( _ sandboxpolicy.Backend = (*PrivilegeBackend)(nil) _ sandboxpolicy.ProtectionAttester = (*PrivilegeBackend)(nil) @@ -195,6 +202,14 @@ func (b *PrivilegeBackend) Replace(ctx context.Context, binding sandboxpolicy.Bi result.Committed = true return result, b.retire(ctx) } + if current != nil && current.revision == plan.Revision && current.combination == combination && + b.instance.Ready() == nil && current.check() == nil { + // Only tracking faults: heal with a new instance, at most every healRetry. + if time.Since(b.lastHeal) < healRetry { + return result, health + } + b.lastHeal = time.Now() + } next := &privilegeBinding{binding: binding, revision: plan.Revision, combination: combination} if current != nil && current.lease != nil && current.check() == nil { next.lease = current.lease diff --git a/pkg/sandboxpolicy/igbackend/privilege_test.go b/pkg/sandboxpolicy/igbackend/privilege_test.go index b6ce5655de..a74089b807 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_test.go +++ b/pkg/sandboxpolicy/igbackend/privilege_test.go @@ -264,6 +264,15 @@ func TestPrivilegeTrackingFaultUnconfirmsAndHeals(t *testing.T) { if err := b.Attest(ctx, privBinding(1), "r1"); err != nil { t.Fatalf("the new instance has no fault: %v", err) } + // A fault that recurs at once does not reload the gadget again. + e.instances[1].faults = map[uint32]uint64{e.instances[1].bound[1]: 1} + if r, err := b.Replace(ctx, privBinding(1), plan); err == nil || r.Committed || !r.ProtectionUnconfirmed || len(e.instances) != 2 { + t.Fatalf("heal within healRetry: %+v %v instances=%d", r, err, len(e.instances)) + } + b.lastHeal = b.lastHeal.Add(-healRetry) + if r, err := b.Replace(ctx, privBinding(1), plan); err != nil || !r.Committed || len(e.instances) != 3 { + t.Fatalf("heal after healRetry: %+v %v instances=%d", r, err, len(e.instances)) + } } func testPodBinding(uid string, cgroup uint64, plan sandboxpolicy.Plan) sandboxpolicy.PodBinding { From 1529726729893222d2dc928101671085822493f1 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Wed, 4 Nov 2026 20:14:02 +0200 Subject: [PATCH 36/54] fix(sandboxpolicy): prepare again on a new namespace; lighter pod poll - A pod that arrived before its new namespace was dropped from slot preparation; a namespace add now prepares again. - The 50 ms pod cgroup poll reuses the pods and plans of the last bind instead of listing pods and compiling policies each time. - A repeated pod binding error from the poll is reported once. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/integration/service.go | 48 +++++++++++++++++-- pkg/sandboxpolicy/integration/service_test.go | 19 +++++++- 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/pkg/sandboxpolicy/integration/service.go b/pkg/sandboxpolicy/integration/service.go index 152c39e527..1c623e6cef 100644 --- a/pkg/sandboxpolicy/integration/service.go +++ b/pkg/sandboxpolicy/integration/service.go @@ -91,6 +91,12 @@ type Service struct { // set while such a pod has no pod cgroup yet, for up to prepareInterval. podFirstSeen map[types.UID]time.Time podPolling bool + // podPods, podQoS and podPlans are the inputs of the last bindPods, + // reused by the poll. podPollErr is the last error the poll reported. + podPods []sandboxpolicy.Binding + podQoS map[types.UID]corev1.PodQOSClass + podPlans map[types.UID]sandboxpolicy.Plan + podPollErr string // podCgroupID returns the cgroup ID of a pod cgroup path (replaced in tests). podCgroupID func(string) (uint64, error) // pods and namespaces are the informers of the node's pods and of all @@ -243,6 +249,12 @@ func (s *Service) watchPods() { } } _, _ = s.namespaces.Core().V1().Namespaces().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + // A pod can arrive before its new namespace: it was dropped, so + // prepare again. + AddFunc: func(any) { + s.podsScheduled.Store(true) + s.ContainerCallback(containercollection.PubSubEvent{}) + }, UpdateFunc: func(old, obj any) { before, ok1 := old.(*corev1.Namespace) after, ok2 := obj.(*corev1.Namespace) @@ -322,7 +334,7 @@ drain: case <-s.wake: s.refresh(runCtx) case <-poll: - s.bindPods(runCtx) + s.pollPods(runCtx) } } }() @@ -385,8 +397,7 @@ func (s *Service) refresh(ctx context.Context) { // exist yet sets podPolling, so it is checked again soon, for up to // prepareInterval after it was first seen. func (s *Service) bindPods(ctx context.Context) { - binder, ok := s.backend.(sandboxpolicy.PodBinder) - if !ok || s.Controller == nil { + if _, ok := s.backend.(sandboxpolicy.PodBinder); !ok || s.Controller == nil { return } s.podPolling = false // set again below when a pod cgroup is missing @@ -400,8 +411,31 @@ func (s *Service) bindPods(ctx context.Context) { } plans, ok := s.Controller.PodPlans(scheduled) if !ok { + s.podPlans = nil return // the policies are not known: keep the pod bindings } + s.podPods, s.podQoS, s.podPlans = scheduled, qos, plans + s.podPollErr = "" + s.bindPodPlans(ctx, scheduled, qos, plans, false) +} + +// pollPods checks again for the pod cgroups that were missing. It reuses the +// pods and plans of the last bindPods, so the 50 ms poll does not list pods +// and compile policies. +func (s *Service) pollPods(ctx context.Context) { + s.podPolling = false + if s.podPlans == nil { + return + } + s.bindPodPlans(ctx, s.podPods, s.podQoS, s.podPlans, true) +} + +func (s *Service) bindPodPlans(ctx context.Context, scheduled []sandboxpolicy.Binding, qos map[types.UID]corev1.PodQOSClass, known map[types.UID]sandboxpolicy.Plan, poll bool) { + binder := s.backend.(sandboxpolicy.PodBinder) + plans := make(map[types.UID]sandboxpolicy.Plan, len(known)) + for k, v := range known { + plans[k] = v + } var pods []sandboxpolicy.PodBinding now := time.Now() firstSeen := map[types.UID]time.Time{} @@ -438,7 +472,13 @@ func (s *Service) bindPods(ctx context.Context) { } s.podFirstSeen, s.podPolling = firstSeen, polling if err := binder.BindPods(ctx, pods); err != nil { - s.report(fmt.Errorf("privilege pod binding: %w", err)) + // The poll runs every 50 ms: report a repeated error once. + if !poll || err.Error() != s.podPollErr { + s.report(fmt.Errorf("privilege pod binding: %w", err)) + } + if poll { + s.podPollErr = err.Error() + } } } diff --git a/pkg/sandboxpolicy/integration/service_test.go b/pkg/sandboxpolicy/integration/service_test.go index 9bba3e0f20..843da0ebfe 100644 --- a/pkg/sandboxpolicy/integration/service_test.go +++ b/pkg/sandboxpolicy/integration/service_test.go @@ -271,11 +271,12 @@ func TestServiceDrainsStartupChurnBeforeReady(t *testing.T) { type podBinderBackend struct { serviceTestBackend pods [][]sandboxpolicy.PodBinding + err error } func (b *podBinderBackend) BindPods(_ context.Context, pods []sandboxpolicy.PodBinding) error { b.pods = append(b.pods, pods) - return nil + return b.err } func TestServiceBindsPodCgroups(t *testing.T) { @@ -316,7 +317,23 @@ func TestServiceBindsPodCgroups(t *testing.T) { if len(last()) != 0 || !s.podPolling { t.Fatalf("missing cgroup: %+v polling=%v", last(), s.podPolling) } + // The poll reuses the last plans: it binds once the cgroup appears, + // and it reports a repeated BindPods error once. id = 77 + s.pollPods(t.Context()) + if got := last(); len(got) != 1 || got[0].CgroupID != 77 || s.podPolling { + t.Fatalf("poll bind: %+v polling=%v", got, s.podPolling) + } + reports := 0 + s.options.OnError = func(error) { reports++ } + backend.err = errors.New("busy") + s.pollPods(t.Context()) + s.pollPods(t.Context()) + backend.err = nil + s.options.OnError = nil + if reports != 1 { + t.Fatalf("a repeated poll error must be reported once: %d", reports) + } s.bindPods(t.Context()) if got := last(); len(got) != 1 || got[0].CgroupID != 77 || got[0].CgroupPath != want || len(got[0].Plan.Policies) != 1 || s.podPolling { t.Fatalf("bind: %+v polling=%v", got, s.podPolling) From 9840fdf379fac1409f3d73923a697a1756efb516 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:06:42 +0200 Subject: [PATCH 37/54] fix(sandboxpolicy): pod plans use the per-class errors of #1054 #1054 now keeps class errors per class name, and a policy fails only on the errors of the shared classes its exec rules use. Pod plans follow the same rule, so a class error in an unrelated policy no longer blocks a pod binding, and the controller builds again with the new base. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/controller.go | 36 +++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/pkg/sandboxpolicy/controller.go b/pkg/sandboxpolicy/controller.go index b8c3f4a4d7..783892e2bd 100644 --- a/pkg/sandboxpolicy/controller.go +++ b/pkg/sandboxpolicy/controller.go @@ -52,9 +52,9 @@ type Controller struct { lastError error bindingError error configError error - // classes and classErr are from the last reconcile (see PodPlans). - classes api.PolicyClasses - classErr error + // classes and classErrors are from the last reconcile (see PodPlans). + classes api.PolicyClasses + classErrors map[string]error } var _ watcher.Adaptor = (*Controller)(nil) @@ -216,7 +216,7 @@ func (c *Controller) PodPlans(containers []Binding) (map[types.UID]Plan, bool) { policies = append(policies, parsePolicy(obj)) } } - classes, classErr := c.classes, c.classErr + classes, classErrors := c.classes, c.classErrors c.mu.Unlock() pods := map[types.UID]Plan{} mixed := map[types.UID]bool{} @@ -224,7 +224,7 @@ func (c *Controller) PodPlans(containers []Binding) (map[types.UID]Plan, bool) { if mixed[b.PodUID] { continue } - plan, ok := podContainerPlan(policies, classes, classErr, b) + plan, ok := podContainerPlan(policies, classes, classErrors, b) if !ok { mixed[b.PodUID] = true delete(pods, b.PodUID) @@ -246,7 +246,7 @@ func (c *Controller) PodPlans(containers []Binding) (map[types.UID]Plan, bool) { // reconcile: an invalid policy that may select it, a class error, too many // policies, or a policy that does not compile without a ContainerProfile // resolver (a policy with sources depends on the profile, so it is refused). -func podContainerPlan(policies []desiredPolicy, classes api.PolicyClasses, classErr error, b Binding) (Plan, bool) { +func podContainerPlan(policies []desiredPolicy, classes api.PolicyClasses, classErrors map[string]error, b Binding) (Plan, bool) { var plan Plan selected := 0 for _, p := range policies { @@ -260,7 +260,7 @@ func podContainerPlan(policies []desiredPolicy, classes api.PolicyClasses, class continue } selected++ - if classErr != nil { + if policyClassError(p, classErrors) != nil { return Plan{}, false } if _, err := Compile(context.Background(), p.identity, p.spec, classes, b, nil); err != nil { @@ -276,6 +276,26 @@ func podContainerPlan(policies []desiredPolicy, classes api.PolicyClasses, class return plan, true } +// policyClassError returns the errors of the shared path classes that p's exec +// rules use, as reconcile does (a class that p defines itself is not shared). +func policyClassError(p desiredPolicy, classErrors map[string]error) error { + if p.spec.Exec == nil { + return nil + } + var err error + for _, rule := range p.spec.Exec.Rules { + for _, name := range rule.Target.PathClass { + if p.spec.Classes != nil { + if _, local := p.spec.Classes.PathClasses[name]; local { + continue + } + } + err = errors.Join(err, classErrors[name]) + } + } + return err +} + // privilegePlans returns the plan of each container, in order (possibly empty). func privilegePlans(policies []desiredPolicy, containers []Binding) []Plan { plans := make([]Plan, len(containers)) @@ -483,7 +503,7 @@ func (c *Controller) reconcile(ctx context.Context) error { classes.PathClasses[name] = paths } } - c.classes, c.classErr = classes, classErr + c.classes, c.classErrors = classes, classErrors policyErrors := make(map[string]error) for _, p := range policies { if p.err != nil { From 5ec67448e8d80356281f83db3286ebb178cd5f0c Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:09:51 +0200 Subject: [PATCH 38/54] fix(sandboxpolicy): translate privilege rules with the new storage API The #1054 base moved to storage v0.0.388: PrivilegeTarget has no path, fromImage or fsType fields any more, and Rule.Origin is a pointer. The translator follows: it rejects actor and pathClass as before, now also rejects except (the gadget has no exceptions, and they were ignored before), and maps a missing origin and an empty one to no origin. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/igbackend/privilege_test.go | 4 +++- pkg/sandboxpolicy/privilege.go | 17 ++++++++++++----- pkg/sandboxpolicy/privilege_test.go | 13 +++++++++++++ 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/pkg/sandboxpolicy/igbackend/privilege_test.go b/pkg/sandboxpolicy/igbackend/privilege_test.go index a74089b807..1bf5abf6b8 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_test.go +++ b/pkg/sandboxpolicy/igbackend/privilege_test.go @@ -68,7 +68,9 @@ func privPlan(revision string, policies ...sandboxpolicy.CompiledPolicy) sandbox return sandboxpolicy.Plan{Revision: revision, Policies: policies} } -var denyMount = api.PrivilegeRule{Rule: api.Rule{Name: "no-mount", Action: api.ActionDeny, Origin: "o"}, Target: api.PrivilegeTarget{Syscall: []string{"mount"}}} +var denyMount = api.PrivilegeRule{Rule: api.Rule{Name: "no-mount", Action: api.ActionDeny, Origin: ptr("o")}, Target: api.PrivilegeTarget{Syscall: []string{"mount"}}} +func ptr(s string) *string { return &s } + var allowMount = api.PrivilegeRule{Rule: api.Rule{Name: "mount-ok", Action: api.ActionAllow}, Target: api.PrivilegeTarget{Syscall: []string{"mount"}}} func TestPrivilegeNodeInstanceSharesSlots(t *testing.T) { diff --git a/pkg/sandboxpolicy/privilege.go b/pkg/sandboxpolicy/privilege.go index 89f9622939..5da3d4159a 100644 --- a/pkg/sandboxpolicy/privilege.go +++ b/pkg/sandboxpolicy/privilege.go @@ -9,8 +9,8 @@ import ( // PrivilegeDocument translates the privilege section of one SandboxPolicy into // the privilege gadget's own policy document. name qualifies rule names in -// events. Actor and path predicates are not supported by the gadget yet and -// are rejected. +// events. Actor, pathClass and except are not supported by the gadget yet and +// are rejected. An explicit empty origin and a missing one both give no origin. // // Several policies on one container merge per the shared merge contract // (pkg/ebpf/gadgets/common/merge/CONTRACT.md). It differs from the CRD design @@ -20,11 +20,18 @@ func PrivilegeDocument(name string, mode api.PolicyMode, section api.PrivilegeSe doc := privpolicy.Document{Name: name, Version: 1, Policy: privpolicy.Policy{Mode: string(mode), Privilege: privpolicy.Section{Default: string(section.Default)}}} for _, r := range section.Rules { t := r.Target - if r.Actor != nil || len(t.Path) > 0 || len(t.PathClass) > 0 || t.FromImage != nil || len(t.FSType) > 0 { - return doc, fmt.Errorf("rule %q: actor and path targets are unsupported in the privilege section", r.Name) + if r.Actor != nil || len(t.PathClass) > 0 { + return doc, fmt.Errorf("rule %q: actor and pathClass targets are unsupported in the privilege section", r.Name) + } + if len(r.Except) > 0 { + return doc, fmt.Errorf("rule %q: exceptions are unsupported in the privilege section", r.Name) + } + origin := "" + if r.Origin != nil { + origin = *r.Origin } doc.Policy.Privilege.Rules = append(doc.Policy.Privilege.Rules, privpolicy.Rule{ - Name: r.Name, Action: string(r.Action), Mode: string(r.Mode), Origin: r.Origin, + Name: r.Name, Action: string(r.Action), Mode: string(r.Mode), Origin: origin, Target: privpolicy.Target{Syscall: t.Syscall, Flags: t.Flags, Capability: t.Capability, MountFSType: t.MountFSType}, }) } diff --git a/pkg/sandboxpolicy/privilege_test.go b/pkg/sandboxpolicy/privilege_test.go index f73852f550..412615fbf7 100644 --- a/pkg/sandboxpolicy/privilege_test.go +++ b/pkg/sandboxpolicy/privilege_test.go @@ -23,3 +23,16 @@ func TestPrivilegeDocumentRejectsFlaglessClone(t *testing.T) { } } } + +func TestPrivilegeDocumentExceptAndOrigin(t *testing.T) { + origin := "catalog/P1" + rule := api.PrivilegeRule{Rule: api.Rule{Name: "r", Action: api.ActionDeny, Origin: &origin}, Target: api.PrivilegeTarget{Syscall: []string{"mount"}}} + doc, err := PrivilegeDocument("p", api.ModeEnforce, api.PrivilegeSection{Default: api.ActionAllow, Rules: []api.PrivilegeRule{rule}}) + if err != nil || doc.Policy.Privilege.Rules[0].Origin != origin { + t.Fatalf("origin: %+v %v", doc.Policy.Privilege.Rules, err) + } + rule.Except = []api.PrivilegeException{{}} + if _, err := PrivilegeDocument("p", api.ModeEnforce, api.PrivilegeSection{Default: api.ActionAllow, Rules: []api.PrivilegeRule{rule}}); err == nil || !strings.Contains(err.Error(), "exceptions") { + t.Fatalf("an exception must be rejected: %v", err) + } +} From a8ef860dbe629847af487c1ca857d7c62a428347 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Thu, 5 Nov 2026 14:10:26 +0200 Subject: [PATCH 39/54] style(sandboxpolicy): gofmt the privilege backend test Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/igbackend/privilege_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/sandboxpolicy/igbackend/privilege_test.go b/pkg/sandboxpolicy/igbackend/privilege_test.go index 1bf5abf6b8..fe0f6eed77 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_test.go +++ b/pkg/sandboxpolicy/igbackend/privilege_test.go @@ -69,6 +69,7 @@ func privPlan(revision string, policies ...sandboxpolicy.CompiledPolicy) sandbox } var denyMount = api.PrivilegeRule{Rule: api.Rule{Name: "no-mount", Action: api.ActionDeny, Origin: ptr("o")}, Target: api.PrivilegeTarget{Syscall: []string{"mount"}}} + func ptr(s string) *string { return &s } var allowMount = api.PrivilegeRule{Rule: api.Rule{Name: "mount-ok", Action: api.ActionAllow}, Target: api.PrivilegeTarget{Syscall: []string{"mount"}}} From 718feedbc50fd210b290f1851f82600b7cb0635a Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:08:07 +0200 Subject: [PATCH 40/54] fix(sandboxpolicy): pod bindings keep their last-good plan Review findings on the pod binding paths: - An edited pod plan whose slot is not ready keeps the last-good pod binding; the pod moves to the new slot once it exists. Only a pod that is not given any more (an authoritative removal) is unbound. - A candidate that does not fit drops only the speculative prepared slots. Active pod bindings are never dropped to make room; when the candidate still does not fit, the new container binding is refused. - Incomplete input (an invalid policy or class, a policy that does not compile, too many policies) gives the pod an empty plan, which keeps its current binding, as the container path keeps its last good plan. Containers with different policies, no privilege policy, or a policy that needs a ContainerProfile still give no pod plan. Each case has a regression test. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/controller.go | 49 +++++++++--- pkg/sandboxpolicy/igbackend/privilege.go | 43 +++------- pkg/sandboxpolicy/igbackend/privilege_test.go | 79 +++++++++++++++++-- pkg/sandboxpolicy/sections_test.go | 19 +++++ pkg/sandboxpolicy/types.go | 3 +- 5 files changed, 143 insertions(+), 50 deletions(-) diff --git a/pkg/sandboxpolicy/controller.go b/pkg/sandboxpolicy/controller.go index 783892e2bd..3429d87fe5 100644 --- a/pkg/sandboxpolicy/controller.go +++ b/pkg/sandboxpolicy/controller.go @@ -202,8 +202,12 @@ func (c *Controller) PrepareScheduled(ctx context.Context, containers []Binding) // binding would also judge the containers that the policies do not select. // A container must also pass the checks that its own binding needs (see // podContainerPlan), so a pod binding never applies a policy that the -// container binding refuses. It returns false when the policies are not known -// (a configuration error). +// container binding refuses. A pod whose input is incomplete (an invalid policy +// or class, a policy that does not compile, too many policies) gets an empty +// plan: keep its current pod binding, as a container binding keeps its last +// good plan. Only an authoritative result (containers with different policies, +// no privilege policy, a policy that needs a ContainerProfile) removes it. It +// returns false when the policies are not known (a configuration error). func (c *Controller) PodPlans(containers []Binding) (map[types.UID]Plan, bool) { c.mu.Lock() if c.configError != nil { @@ -220,12 +224,16 @@ func (c *Controller) PodPlans(containers []Binding) (map[types.UID]Plan, bool) { c.mu.Unlock() pods := map[types.UID]Plan{} mixed := map[types.UID]bool{} + keep := map[types.UID]bool{} for _, b := range containers { + plan, state := podContainerPlan(policies, classes, classErrors, b) + if state == podPlanKeep { + keep[b.PodUID] = true + } if mixed[b.PodUID] { continue } - plan, ok := podContainerPlan(policies, classes, classErrors, b) - if !ok { + if state != podPlanReady { mixed[b.PodUID] = true delete(pods, b.PodUID) continue @@ -238,15 +246,26 @@ func (c *Controller) PodPlans(containers []Binding) (map[types.UID]Plan, bool) { } pods[b.PodUID] = plan } + for uid := range keep { + pods[uid] = Plan{} // incomplete input: keep the current binding + } return pods, true } +type podPlanState int + +const ( + podPlanNone podPlanState = iota // no pod binding (authoritative) + podPlanReady // the pod plan + podPlanKeep // incomplete input: keep the current binding +) + // podContainerPlan returns the privilege plan of a container for a pod // binding. It is false when the container's own binding would fail, as in // reconcile: an invalid policy that may select it, a class error, too many // policies, or a policy that does not compile without a ContainerProfile // resolver (a policy with sources depends on the profile, so it is refused). -func podContainerPlan(policies []desiredPolicy, classes api.PolicyClasses, classErrors map[string]error, b Binding) (Plan, bool) { +func podContainerPlan(policies []desiredPolicy, classes api.PolicyClasses, classErrors map[string]error, b Binding) (Plan, podPlanState) { var plan Plan selected := 0 for _, p := range policies { @@ -254,26 +273,34 @@ func podContainerPlan(policies []desiredPolicy, classes api.PolicyClasses, class if p.identity.Kind == "SandboxPolicy" && p.identity.Namespace != b.Namespace { continue } - return Plan{}, false + return Plan{}, podPlanKeep } if p.spec.Mode == api.ModeOff || !p.matches(b) { continue } selected++ + if len(p.spec.Sources) > 0 { + // Needs the ContainerProfile, which only the container binding + // resolves. + return Plan{}, podPlanNone + } if policyClassError(p, classErrors) != nil { - return Plan{}, false + return Plan{}, podPlanKeep } if _, err := Compile(context.Background(), p.identity, p.spec, classes, b, nil); err != nil { - return Plan{}, false + return Plan{}, podPlanKeep } if p.spec.Privilege != nil { plan.Policies = append(plan.Policies, CompiledPolicy{Identity: p.identity, Mode: p.spec.Mode, Privilege: p.spec.Privilege.DeepCopy()}) } } - if selected > MaxPoliciesPerBinding || len(plan.Policies) == 0 { - return Plan{}, false + if selected > MaxPoliciesPerBinding { + return Plan{}, podPlanKeep + } + if len(plan.Policies) == 0 { + return Plan{}, podPlanNone } - return plan, true + return plan, podPlanReady } // policyClassError returns the errors of the shared path classes that p's exec diff --git a/pkg/sandboxpolicy/igbackend/privilege.go b/pkg/sandboxpolicy/igbackend/privilege.go index 556dc5660c..dec4134eb2 100644 --- a/pkg/sandboxpolicy/igbackend/privilege.go +++ b/pkg/sandboxpolicy/igbackend/privilege.go @@ -247,10 +247,11 @@ func (b *PrivilegeBackend) Replace(ctx context.Context, binding sandboxpolicy.Bi } extra := map[string][]sandboxpolicy.CompiledPolicy{combination: policies} err := b.replaceInstance(ctx, b.prepared, extra, k, next, true) - if err != nil && (len(b.prepared) > 0 || b.podsAddSlots(extra)) { - // The prepared set and the pod bindings may not fit; never let them - // block a real container. BindPods binds the pods again later. - err = b.replaceInstance(ctx, nil, extra, k, next, false) + if err != nil && len(b.prepared) > 0 { + // The speculative prepared slots may not fit: drop only those. The + // active pod bindings stay; if the candidate still does not fit, the + // new binding is refused. + err = b.replaceInstance(ctx, nil, extra, k, next, true) } if err != nil { return fail(err) @@ -341,11 +342,12 @@ func (b *PrivilegeBackend) Prepare(ctx context.Context, plans []sandboxpolicy.Pl } // BindPods binds the given pod cgroups to the slot of their plan and unbinds -// the pod cgroups that are not given. A pod whose slot does not exist yet is -// skipped: Prepare creates the slot, and the next call binds the pod. A pod -// whose plan changes moves to the new slot in one map write; when the new slot -// does not exist yet, the pod is unbound, so a removed or edited policy never -// stays active at pod level. +// the pod cgroups that are not given (an authoritative removal). A pod whose +// slot does not exist yet is skipped: Prepare creates the slot, and the next +// call binds the pod. A pod whose plan changes keeps its last-good binding +// until the new slot is ready, then moves to it in one map write. A pod given +// with an empty plan (incomplete input, see Controller.PodPlans) keeps its +// current binding. func (b *PrivilegeBackend) BindPods(ctx context.Context, pods []sandboxpolicy.PodBinding) error { b.mu.Lock() defer b.mu.Unlock() @@ -361,10 +363,7 @@ func (b *PrivilegeBackend) BindPods(ctx context.Context, pods []sandboxpolicy.Po for uid, current := range b.pods { next, ok := wanted[uid] if ok && next.CgroupID == current.pod.CgroupID && next.CgroupPath == current.pod.CgroupPath { - combination, _ := Combination(next.Plan) - if _, slot := b.slots[combination]; slot || combination == current.combination { - continue - } + continue // the same pod cgroup: kept, or moved below } // Not wanted, or a new cgroup for the pod: remove the old identity. // An instance that is not ready protects nothing. @@ -396,24 +395,6 @@ func (b *PrivilegeBackend) BindPods(ctx context.Context, pods []sandboxpolicy.Po return errors.Join(append(errs, b.retire(ctx))...) } -// podsAddSlots reports whether a pod binding needs a combination that no -// container binding, and not extra, needs. -func (b *PrivilegeBackend) podsAddSlots(extra map[string][]sandboxpolicy.CompiledPolicy) bool { - used := map[string]bool{} - for c := range extra { - used[c] = true - } - for _, v := range b.bindings { - used[v.combination] = true - } - for _, p := range b.pods { - if !used[p.combination] { - return true - } - } - return false -} - // replaceInstance starts a candidate with every combination in use or prepared // (plus extra), binds every container (plus next, under nextKey) and, with // withPods, every pod cgroup, and only then stops the old instance. Without diff --git a/pkg/sandboxpolicy/igbackend/privilege_test.go b/pkg/sandboxpolicy/igbackend/privilege_test.go index fe0f6eed77..3b536054cd 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_test.go +++ b/pkg/sandboxpolicy/igbackend/privilege_test.go @@ -41,12 +41,16 @@ type fakePrivilegeEngine struct { instances []*fakePrivilegeInstance configs []privruntime.Config startErr error + failIf func(privruntime.Config) bool } func (e *fakePrivilegeEngine) Start(_ context.Context, _ string, c privruntime.Config) (PrivilegeInstance, error) { if e.startErr != nil { return nil, e.startErr } + if e.failIf != nil && e.failIf(c) { + return nil, errors.New("does not fit") + } i := &fakePrivilegeInstance{bound: map[uint64]uint32{}} for _, b := range c.InitialBindings { i.bound[b.Container.CgroupID] = b.Slot @@ -345,7 +349,7 @@ func TestPrivilegePodBindings(t *testing.T) { } } -func TestPrivilegePodUnboundWhenNewSlotIsMissing(t *testing.T) { +func TestPrivilegePodKeepsLastGoodUntilNewSlotIsReady(t *testing.T) { e := &fakePrivilegeEngine{} b, _ := NewPrivilege(e, nil, "n") ctx := context.Background() @@ -356,22 +360,83 @@ func TestPrivilegePodUnboundWhenNewSlotIsMissing(t *testing.T) { if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, p1)}); err != nil { t.Fatal(err) } - // An edited policy without a slot yet: the old policy must not stay. + old := e.instances[0].bound[100] + // An edited policy without a slot yet: the last-good binding stays. edited := privPlan("r2", privPolicy("p1", api.ActionDeny)) if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, edited)}); err != nil { t.Fatal(err) } - if _, ok := e.instances[0].bound[100]; ok { - t.Fatal("a pod whose new slot is missing must be unbound") + if slot, ok := e.instances[0].bound[100]; !ok || slot != old { + t.Fatal("an edit without a ready slot must keep the last-good pod binding") + } + // Incomplete input (an empty plan) keeps it too. + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, sandboxpolicy.Plan{})}); err != nil { + t.Fatal(err) + } + if _, ok := e.instances[0].bound[100]; !ok { + t.Fatal("incomplete input must keep the pod binding") } - if err := b.Prepare(ctx, []sandboxpolicy.Plan{edited}); err != nil { + // The new slot is ready: the pod moves to it. + if err := b.Prepare(ctx, []sandboxpolicy.Plan{p1, edited}); err != nil { t.Fatal(err) } + last := e.instances[len(e.instances)-1] if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, edited)}); err != nil { t.Fatal(err) } - if _, ok := e.instances[len(e.instances)-1].bound[100]; !ok { - t.Fatal("the pod must be bound once the new slot exists") + if b.pods["a"] == nil || b.pods["a"].combination != func() string { c, _ := Combination(edited); return c }() { + t.Fatal("the pod must move to the new slot once it is ready") + } + if _, ok := last.bound[100]; !ok { + t.Fatal("the pod must be bound in the running instance") + } + // An authoritative removal unbinds it. + if err := b.BindPods(ctx, nil); err != nil { + t.Fatal(err) + } + if _, ok := last.bound[100]; ok { + t.Fatal("a pod that is not given must be unbound") + } +} + +// A candidate that does not fit may drop only the speculative prepared slots, +// never the active pod bindings; without prepared slots the new container is +// refused and the running instance keeps protecting the pods. +func TestPrivilegeRetryNeverDropsPodBindings(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + podPlan := privPlan("r1", privPolicy("pod", api.ActionAllow, denyMount)) + if err := b.Prepare(ctx, []sandboxpolicy.Plan{podPlan}); err != nil { + t.Fatal(err) + } + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, podPlan)}); err != nil { + t.Fatal(err) + } + spare := privPlan("r2", privPolicy("spare", api.ActionDeny)) + if err := b.Prepare(ctx, []sandboxpolicy.Plan{podPlan, spare}); err != nil { + t.Fatal(err) + } + // Room for two slots only. + e.failIf = func(c privruntime.Config) bool { return c.Policy.Slots > 2 } + if r, err := b.Replace(ctx, privBinding(1), privPlan("r3", privPolicy("new", api.ActionDeny))); err != nil || !r.Committed { + t.Fatalf("the retry without prepared slots must fit: %+v %v", r, err) + } + carried := false + for _, initial := range e.configs[len(e.configs)-1].InitialBindings { + carried = carried || initial.Container.CgroupID == 100 + } + if !carried { + t.Fatal("the retry must keep the pod binding") + } + // No prepared slots left to drop: a container that needs a third slot is refused. + e.failIf = func(c privruntime.Config) bool { return c.Policy.Slots > 2 } + running := e.instances[len(e.instances)-1] + if r, err := b.Replace(ctx, privBinding(2), privPlan("r4", privPolicy("third", api.ActionDeny))); err == nil || r.Committed { + t.Fatalf("a candidate that does not fit must be refused: %+v %v", r, err) + } + if _, bound := running.bound[100]; running.stops != 0 || !bound || b.pods["a"] == nil { + t.Fatal("the running instance must keep protecting the pod") } } diff --git a/pkg/sandboxpolicy/sections_test.go b/pkg/sandboxpolicy/sections_test.go index 6ff9a75b2c..35b528f2d6 100644 --- a/pkg/sandboxpolicy/sections_test.go +++ b/pkg/sandboxpolicy/sections_test.go @@ -332,6 +332,25 @@ func TestControllerPodPlans(t *testing.T) { } } +// An invalid policy that may select the pod makes its input incomplete: the +// pod gets an empty plan (keep the current binding), not a removal. +func TestControllerPodPlansKeepOnIncompleteInput(t *testing.T) { + ctx := context.Background() + priv := policy("priv", "ns") + priv.Spec.Privilege = privilegeSection(api.ActionDeny) + broken := policy("broken", "ns") + broken.Spec.Containers = nil // invalid: containers must not be empty + c, _, _, _ := setup(t, priv, broken) + _ = c.InitialSync(ctx) + pod := binding() + pod.ContainerID, pod.CgroupID, pod.PodUID = "", 0, "pod" + plans, ok := c.PodPlans([]Binding{pod}) + plan, present := plans["pod"] + if !ok || !present || len(plan.Policies) != 0 { + t.Fatalf("incomplete input must keep the pod binding (empty plan): %+v", plans) + } +} + func TestControllerPodPlansNeedContainerEligibility(t *testing.T) { ctx := context.Background() priv := policy("priv", "ns") diff --git a/pkg/sandboxpolicy/types.go b/pkg/sandboxpolicy/types.go index bb5aa4a325..21c292c1d9 100644 --- a/pkg/sandboxpolicy/types.go +++ b/pkg/sandboxpolicy/types.go @@ -124,7 +124,8 @@ type PodBinding struct { } // PodBinder optionally binds pod cgroups. BindPods receives the complete set: -// a pod binding that is not in it is removed. Like SlotPreparer, it is a hint +// a pod binding that is not in it is removed. A pod given with an empty Plan +// keeps its current binding (its input is incomplete). Like SlotPreparer, it is a hint // that changes no container binding. type PodBinder interface { BindPods(context.Context, []PodBinding) error From 5c2a65e7fd02279b83554efe637e23fd7dd91411 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Sun, 8 Nov 2026 11:37:12 +0200 Subject: [PATCH 41/54] fix(sandboxpolicy): pass privilege origins and the default mode to the gadget The translator now keeps origin presence (a missing origin stays nil, an explicit "" stays "") for rules and the section default, and passes the section defaultMode, which it dropped before. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/privilege.go | 13 ++++++------- pkg/sandboxpolicy/privilege_test.go | 27 ++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/pkg/sandboxpolicy/privilege.go b/pkg/sandboxpolicy/privilege.go index 5da3d4159a..06b285210d 100644 --- a/pkg/sandboxpolicy/privilege.go +++ b/pkg/sandboxpolicy/privilege.go @@ -10,14 +10,17 @@ import ( // PrivilegeDocument translates the privilege section of one SandboxPolicy into // the privilege gadget's own policy document. name qualifies rule names in // events. Actor, pathClass and except are not supported by the gadget yet and -// are rejected. An explicit empty origin and a missing one both give no origin. +// are rejected. Origins keep their presence: an explicit "" is not a missing +// origin. // // Several policies on one container merge per the shared merge contract // (pkg/ebpf/gadgets/common/merge/CONTRACT.md). It differs from the CRD design // text (sandbox-policy-crd.md, "Multiple policies matching one pod merge"): an // audit deny rule never weakens another policy's enforce default deny. func PrivilegeDocument(name string, mode api.PolicyMode, section api.PrivilegeSection) (privpolicy.Document, error) { - doc := privpolicy.Document{Name: name, Version: 1, Policy: privpolicy.Policy{Mode: string(mode), Privilege: privpolicy.Section{Default: string(section.Default)}}} + doc := privpolicy.Document{Name: name, Version: 1, Policy: privpolicy.Policy{Mode: string(mode), Privilege: privpolicy.Section{ + Default: string(section.Default), DefaultMode: string(section.DefaultMode), DefaultOrigin: section.DefaultOrigin, + }}} for _, r := range section.Rules { t := r.Target if r.Actor != nil || len(t.PathClass) > 0 { @@ -26,12 +29,8 @@ func PrivilegeDocument(name string, mode api.PolicyMode, section api.PrivilegeSe if len(r.Except) > 0 { return doc, fmt.Errorf("rule %q: exceptions are unsupported in the privilege section", r.Name) } - origin := "" - if r.Origin != nil { - origin = *r.Origin - } doc.Policy.Privilege.Rules = append(doc.Policy.Privilege.Rules, privpolicy.Rule{ - Name: r.Name, Action: string(r.Action), Mode: string(r.Mode), Origin: origin, + Name: r.Name, Action: string(r.Action), Mode: string(r.Mode), Origin: r.Origin, Target: privpolicy.Target{Syscall: t.Syscall, Flags: t.Flags, Capability: t.Capability, MountFSType: t.MountFSType}, }) } diff --git a/pkg/sandboxpolicy/privilege_test.go b/pkg/sandboxpolicy/privilege_test.go index 412615fbf7..ae74e96018 100644 --- a/pkg/sandboxpolicy/privilege_test.go +++ b/pkg/sandboxpolicy/privilege_test.go @@ -28,7 +28,7 @@ func TestPrivilegeDocumentExceptAndOrigin(t *testing.T) { origin := "catalog/P1" rule := api.PrivilegeRule{Rule: api.Rule{Name: "r", Action: api.ActionDeny, Origin: &origin}, Target: api.PrivilegeTarget{Syscall: []string{"mount"}}} doc, err := PrivilegeDocument("p", api.ModeEnforce, api.PrivilegeSection{Default: api.ActionAllow, Rules: []api.PrivilegeRule{rule}}) - if err != nil || doc.Policy.Privilege.Rules[0].Origin != origin { + if err != nil || doc.Policy.Privilege.Rules[0].Origin == nil || *doc.Policy.Privilege.Rules[0].Origin != origin { t.Fatalf("origin: %+v %v", doc.Policy.Privilege.Rules, err) } rule.Except = []api.PrivilegeException{{}} @@ -36,3 +36,28 @@ func TestPrivilegeDocumentExceptAndOrigin(t *testing.T) { t.Fatalf("an exception must be rejected: %v", err) } } + +// Origin presence and the default mode reach the gadget document: a missing +// origin stays nil, an explicit "" stays "". +func TestPrivilegeDocumentOriginPresenceAndDefaults(t *testing.T) { + empty := "" + section := api.PrivilegeSection{Default: api.ActionDeny, DefaultMode: api.RuleAudit, DefaultOrigin: &empty, Rules: []api.PrivilegeRule{ + {Rule: api.Rule{Name: "none", Action: api.ActionDeny}, Target: api.PrivilegeTarget{Syscall: []string{"mount"}}}, + {Rule: api.Rule{Name: "blank", Action: api.ActionDeny, Origin: &empty}, Target: api.PrivilegeTarget{Syscall: []string{"bpf"}}}, + }} + doc, err := PrivilegeDocument("p", api.ModeEnforce, section) + if err != nil { + t.Fatal(err) + } + s := doc.Policy.Privilege + if s.DefaultMode != "audit" || s.DefaultOrigin == nil || *s.DefaultOrigin != "" { + t.Fatalf("defaults: mode %q origin %v", s.DefaultMode, s.DefaultOrigin) + } + if s.Rules[0].Origin != nil || s.Rules[1].Origin == nil || *s.Rules[1].Origin != "" { + t.Fatalf("rule origins: %v %v", s.Rules[0].Origin, s.Rules[1].Origin) + } + section.DefaultOrigin = nil + if doc, _ = PrivilegeDocument("p", api.ModeEnforce, section); doc.Policy.Privilege.DefaultOrigin != nil { + t.Fatal("a missing default origin must stay missing") + } +} From 47c6386d2db8213af214ef88106e19a59af0d75d Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:29:17 +0200 Subject: [PATCH 42/54] feat(sandboxpolicy): report the applied generation and a true tier per node Node status now carries appliedGeneration, the oldest generation still installed, so a rejected revision no longer hides which one enforces. The BPF LSM probe ran only for the privilege section, but exec also enforces through BPF LSM, so an exec policy reported Ready and tier bpf-lsm on nodes where nothing runs. The probe result now applies to every section and such nodes report tier none. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- docs/features/sandbox-policy.md | 7 +++++ pkg/sandboxpolicy/controller.go | 4 +++ pkg/sandboxpolicy/controller_test.go | 9 ++++++ pkg/sandboxpolicy/igbackend/lsm.go | 9 +++--- pkg/sandboxpolicy/integration/service.go | 5 ++-- pkg/sandboxpolicy/sections_test.go | 36 ++++++++++++++++++++++++ pkg/sandboxpolicy/status.go | 15 ++++++++-- pkg/sandboxpolicy/status_test.go | 3 ++ 8 files changed, 80 insertions(+), 8 deletions(-) diff --git a/docs/features/sandbox-policy.md b/docs/features/sandbox-policy.md index 95200b0f3f..77aee6c42b 100644 --- a/docs/features/sandbox-policy.md +++ b/docs/features/sandbox-policy.md @@ -68,6 +68,13 @@ This change does not grant chart RBAC, change a chart version or store policy in SQLite/files. Kubernetes owns the CRD objects. Per-policy/node status resources are output; the agent does not write parent aggregate status. +Each report carries `appliedGeneration`, the oldest policy generation still +installed on the node. It stays on the last-good generation while a newer +revision is rejected, and is absent when no binding enforces the policy. At +startup the agent reads the host's active LSM list once. Every section enforces +through BPF LSM, so when `bpf` is missing every report says `tier: none`, and +every enabled policy reports `Ready=False` with reason `BPFLSMInactive`. + ## Reconciliation and failure behavior Configuration watch events are **refresh hints**, not trusted queued payloads. diff --git a/pkg/sandboxpolicy/controller.go b/pkg/sandboxpolicy/controller.go index 3429d87fe5..7c6868f431 100644 --- a/pkg/sandboxpolicy/controller.go +++ b/pkg/sandboxpolicy/controller.go @@ -29,6 +29,10 @@ var NodeStatusGVR = api.SchemeGroupVersion.WithResource("sandboxpolicynodestatus type Options struct { Node api.NodeReference AgentNamespace string + // EnforcementUnavailable says why no section can enforce or audit on this + // node (for example, BPF LSM is not active). It comes from a probe at startup; + // nil means the probe passed. + EnforcementUnavailable error } type installed struct { diff --git a/pkg/sandboxpolicy/controller_test.go b/pkg/sandboxpolicy/controller_test.go index 5c1f234d8e..5f06cbcc35 100644 --- a/pkg/sandboxpolicy/controller_test.go +++ b/pkg/sandboxpolicy/controller_test.go @@ -179,6 +179,12 @@ func TestInvalidRevisionRetainsLastGoodWhileClusterUpdates(t *testing.T) { if stale.Conditions[0].Status != metav1.ConditionFalse || !stale.Stale || len(stale.Conditions) != 2 || !strings.Contains(stale.Conditions[1].Message, "generation=1") { t.Fatalf("last-good status missing: %+v", stale) } + if stale.ObservedGeneration != 2 || stale.AppliedGeneration != 1 { + t.Fatalf("applied generation must stay on last-good: %+v", stale) + } + if applied := writer.reports["cluster"].Status.AppliedGeneration; applied != 2 { + t.Fatalf("cluster applied generation %d, want 2", applied) + } } func TestClassErrorOnlyBlocksDependentPolicy(t *testing.T) { @@ -342,6 +348,9 @@ func TestInitialSyncUpdateLastGoodAndWithdrawal(t *testing.T) { if report.Status.Conditions[0].Status != metav1.ConditionFalse || report.Status.ObservedGeneration != 2 || !report.Status.Stale || !strings.Contains(report.Status.Conditions[0].Message, "last-good") { t.Fatalf("misleading report: %+v", report.Status) } + if report.Status.AppliedGeneration != 1 { + t.Fatalf("applied generation %d, want last-good 1", report.Status.AppliedGeneration) + } b.fail = false if err := c.Reconcile(ctx); err != nil { t.Fatal(err) diff --git a/pkg/sandboxpolicy/igbackend/lsm.go b/pkg/sandboxpolicy/igbackend/lsm.go index a5ac7e8867..d2e4e49126 100644 --- a/pkg/sandboxpolicy/igbackend/lsm.go +++ b/pkg/sandboxpolicy/igbackend/lsm.go @@ -14,21 +14,22 @@ import ( // CheckBPFLSM returns an EnforcementUnavailableError unless "bpf" is in the // host's active LSM list. Without it, BPF LSM programs attach but never run // (for example the default AKS Ubuntu and Azure Linux images), so enforcement -// would be reported while nothing is enforced. +// would be reported while nothing is enforced. Every section enforces through +// BPF LSM, so the result applies to all of them. func CheckBPFLSM(hostRoot string) error { path := filepath.Join(hostRoot, "sys/kernel/security/lsm") data, err := os.ReadFile(path) if errors.Is(err, fs.ErrNotExist) { - return &sandboxpolicy.EnforcementUnavailableError{Section: "privilege", Reason: "BPFLSMInactive", Detail: fmt.Sprintf("%s is missing (securityfs not mounted)", path)} + return &sandboxpolicy.EnforcementUnavailableError{Section: "sandbox policy", Reason: "BPFLSMInactive", Detail: fmt.Sprintf("%s is missing (securityfs not mounted)", path)} } if err != nil { - return &sandboxpolicy.EnforcementUnavailableError{Section: "privilege", Reason: "BPFLSMInactive", Detail: fmt.Sprintf("reading %s: %v", path, err)} + return &sandboxpolicy.EnforcementUnavailableError{Section: "sandbox policy", Reason: "BPFLSMInactive", Detail: fmt.Sprintf("reading %s: %v", path, err)} } for _, lsm := range strings.Split(strings.TrimSpace(string(data)), ",") { if lsm == "bpf" { return nil } } - return &sandboxpolicy.EnforcementUnavailableError{Section: "privilege", Reason: "BPFLSMInactive", + return &sandboxpolicy.EnforcementUnavailableError{Section: "sandbox policy", Reason: "BPFLSMInactive", Detail: fmt.Sprintf("BPF LSM is not active (active LSMs: %s); add bpf to the lsm= kernel parameter", strings.TrimSpace(string(data)))} } diff --git a/pkg/sandboxpolicy/integration/service.go b/pkg/sandboxpolicy/integration/service.go index af9195ca63..d3806d0839 100644 --- a/pkg/sandboxpolicy/integration/service.go +++ b/pkg/sandboxpolicy/integration/service.go @@ -177,8 +177,9 @@ func NewService(ctx context.Context, options ServiceOptions) (*Service, error) { } var privilege sandboxpolicy.Backend var privilegeUnavailable error + lsmUnavailable := igbackend.CheckBPFLSM(host.HostRoot) if options.PrivilegeImageName != "" { - privilegeUnavailable = igbackend.CheckBPFLSM(host.HostRoot) + privilegeUnavailable = lsmUnavailable } if options.PrivilegeImageName != "" && privilegeUnavailable == nil { privilegeEngine, err := igbackend.NewNativePrivilegeEngine(r, kubemanager.NewKubeManager(options.Containers, options.Tracers), store, options.PrivilegeImageName) @@ -197,7 +198,7 @@ func NewService(ctx context.Context, options ServiceOptions) (*Service, error) { _ = s.closeRuntime() return nil, err } - s.Controller, err = sandboxpolicy.NewController(options.Dynamic, s.backend, options.Sources, readinessWriter{inner: &sandboxpolicy.DynamicStatusWriter{Client: options.Dynamic}, observed: &s.inputEpoch, snapshot: &s.snapshotEpoch}, sandboxpolicy.Options{Node: api.NodeReference{Name: node.Name, UID: node.UID}, AgentNamespace: options.AgentNamespace}) + s.Controller, err = sandboxpolicy.NewController(options.Dynamic, s.backend, options.Sources, readinessWriter{inner: &sandboxpolicy.DynamicStatusWriter{Client: options.Dynamic}, observed: &s.inputEpoch, snapshot: &s.snapshotEpoch}, sandboxpolicy.Options{Node: api.NodeReference{Name: node.Name, UID: node.UID}, AgentNamespace: options.AgentNamespace, EnforcementUnavailable: lsmUnavailable}) if err != nil { _ = s.backend.Close(ctx) _ = s.closeRuntime() diff --git a/pkg/sandboxpolicy/sections_test.go b/pkg/sandboxpolicy/sections_test.go index 35b528f2d6..0d4d702297 100644 --- a/pkg/sandboxpolicy/sections_test.go +++ b/pkg/sandboxpolicy/sections_test.go @@ -270,6 +270,9 @@ func TestPrivilegeUnavailableIsReportedAndRefused(t *testing.T) { if ready.Status != metav1.ConditionFalse || ready.Reason != "BPFLSMInactive" { t.Fatalf("ready condition %+v", ready) } + if report.Status.Tier != "none" { + t.Fatalf("tier %q, want none", report.Status.Tier) + } if _, err := sections.Replace(ctx, Binding{ContainerID: "c"}, sectionPlan("r", CompiledPolicy{Privilege: privilegeSection(api.ActionDeny)})); !errors.Is(err, lsm) { t.Fatalf("privilege binding must be refused: %v", err) } @@ -374,3 +377,36 @@ func TestControllerPodPlansNeedContainerEligibility(t *testing.T) { t.Fatalf("only the pod without a sourced policy has a pod plan: %+v", plans) } } + +// Every section enforces through BPF LSM. Without it, an exec-only policy must +// not report Ready or a bpf-lsm tier either. +func TestNodeWithoutBPFLSMReportsTierNone(t *testing.T) { + ctx := context.Background() + execOnly := policy("exec-only", "ns") + off := policy("off", "ns") + off.Spec.Mode = api.ModeOff + c, _, w, _ := setup(t, execOnly, off) + lsm := &EnforcementUnavailableError{Section: "sandbox policy", Reason: "BPFLSMInactive", Detail: "BPF LSM is not active"} + c.options.EnforcementUnavailable = lsm + if err := c.InitialSync(ctx); err != nil && !errors.Is(err, lsm) { + t.Fatalf("initial sync: %v", err) + } + enabled := w.reports[string(execOnly.UID)] + if enabled == nil || enabled.Status.Tier != "none" || enabled.Status.Conditions[0].Status != metav1.ConditionFalse || enabled.Status.Conditions[0].Reason != "BPFLSMInactive" { + t.Fatalf("exec-only report %+v", enabled) + } + disabled := w.reports[string(off.UID)] + if disabled == nil || disabled.Status.Tier != "none" || disabled.Status.Conditions[0].Status != metav1.ConditionTrue { + t.Fatalf("off policy report %+v", disabled) + } +} + +func TestNodeWithBPFLSMReportsTierBPFLSM(t *testing.T) { + c, _, w, _ := setup(t, policy("exec-only", "ns")) + if err := c.InitialSync(context.Background()); err != nil { + t.Fatal(err) + } + if r := w.reports["exec-only"]; r == nil || r.Status.Tier != "bpf-lsm" { + t.Fatalf("report %+v", r) + } +} diff --git a/pkg/sandboxpolicy/status.go b/pkg/sandboxpolicy/status.go index 14e6db3097..d1f338be2d 100644 --- a/pkg/sandboxpolicy/status.go +++ b/pkg/sandboxpolicy/status.go @@ -40,6 +40,10 @@ func (c *Controller) report(p desiredPolicy, failure error) *api.SandboxPolicyNo continue } active++ + // The oldest generation still installed is the one this node can vouch for. + if status.AppliedGeneration == 0 || compiled.Identity.Generation < status.AppliedGeneration { + status.AppliedGeneration = compiled.Identity.Generation + } revisions[fmt.Sprintf("generation=%d revision=%s", compiled.Identity.Generation, compiled.Revision)] = true if compiled.Identity.Generation != p.identity.Generation { stale = true @@ -53,9 +57,16 @@ func (c *Controller) report(p desiredPolicy, failure error) *api.SandboxPolicyNo if p.spec.Mode == api.ModeOff { message = "Policy is off" } - if p.spec.Privilege != nil && p.spec.Mode != api.ModeOff { - // Without BPF LSM the programs attach but never run: never report Ready. + // Without BPF LSM the programs attach but never run: nothing enforces or + // audits, so the tier is none and an enabled policy is never Ready. + if c.options.EnforcementUnavailable != nil { + status.Tier = "none" + if p.spec.Mode != api.ModeOff { + failure = errors.Join(failure, c.options.EnforcementUnavailable) + } + } else if p.spec.Privilege != nil && p.spec.Mode != api.ModeOff { if err := c.privilegeUnavailable(); err != nil { + status.Tier = "none" failure = errors.Join(failure, err) } } diff --git a/pkg/sandboxpolicy/status_test.go b/pkg/sandboxpolicy/status_test.go index 2e9be7575e..2f11636a4f 100644 --- a/pkg/sandboxpolicy/status_test.go +++ b/pkg/sandboxpolicy/status_test.go @@ -22,6 +22,9 @@ func TestStatusIdentityBoundsAndUnprotected(t *testing.T) { if !r.Status.Truncated || len(r.Status.Error) != api.MaxStatusErrorLength || r.Status.Conditions[0].Status != metav1.ConditionFalse || !strings.Contains(r.Status.Conditions[0].Message, "no last-good") { t.Fatalf("incorrect failure: %+v", r.Status) } + if r.Status.AppliedGeneration != 0 { + t.Fatalf("applied generation %d with no installed binding", r.Status.AppliedGeneration) + } } func parseTypedPolicy(t *testing.T, p *api.SandboxPolicy) desiredPolicy { From bc0228afab528e41f65a53c929e852b323bb19d8 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:31:09 +0200 Subject: [PATCH 43/54] fix(sandboxpolicy): keep tier and applied generation true in edge cases Without BPF LSM, installed bindings enforce nothing, so the report no longer names an applied generation. A refused privilege section no longer reports tier none while last-good exec bindings still enforce. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/status.go | 6 ++++- pkg/sandboxpolicy/status_test.go | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/pkg/sandboxpolicy/status.go b/pkg/sandboxpolicy/status.go index d1f338be2d..a4f66f33e4 100644 --- a/pkg/sandboxpolicy/status.go +++ b/pkg/sandboxpolicy/status.go @@ -61,12 +61,16 @@ func (c *Controller) report(p desiredPolicy, failure error) *api.SandboxPolicyNo // audits, so the tier is none and an enabled policy is never Ready. if c.options.EnforcementUnavailable != nil { status.Tier = "none" + status.AppliedGeneration = 0 // installed, but no generation is enforced if p.spec.Mode != api.ModeOff { failure = errors.Join(failure, c.options.EnforcementUnavailable) } } else if p.spec.Privilege != nil && p.spec.Mode != api.ModeOff { if err := c.privilegeUnavailable(); err != nil { - status.Tier = "none" + // Last-good exec bindings still enforce through BPF LSM. + if active == 0 { + status.Tier = "none" + } failure = errors.Join(failure, err) } } diff --git a/pkg/sandboxpolicy/status_test.go b/pkg/sandboxpolicy/status_test.go index 2f11636a4f..0a78ec34aa 100644 --- a/pkg/sandboxpolicy/status_test.go +++ b/pkg/sandboxpolicy/status_test.go @@ -77,3 +77,41 @@ func TestDynamicStatusWriterCreateHeartbeatAndIdentity(t *testing.T) { } } } + +func TestAppliedGenerationIsOldestConfirmedBinding(t *testing.T) { + c, _, _, _ := setup(t) + p := parseTypedPolicy(t, policy("policy", "ns")) + p.identity.Generation = 3 + installedAt := func(container string, generation int64, unconfirmed bool) { + b := binding() + b.ContainerID = container + c.active[bindingKey(b)] = installed{binding: b, protectionUnconfirmed: unconfirmed, plan: Plan{Policies: []CompiledPolicy{{Identity: PolicyIdentity{UID: p.identity.UID, Generation: generation}, Revision: container}}}} + } + installedAt("new", 3, false) + installedAt("old", 2, false) + installedAt("unconfirmed", 1, true) // cannot vouch for it + if r := c.report(p, nil); r.Status.AppliedGeneration != 2 || r.Status.Tier != "bpf-lsm" { + t.Fatalf("status %+v, want applied generation 2", r.Status) + } + c.options.EnforcementUnavailable = &EnforcementUnavailableError{Section: "sandbox policy", Reason: "BPFLSMInactive", Detail: "BPF LSM is not active"} + if r := c.report(p, nil); r.Status.AppliedGeneration != 0 || r.Status.Tier != "none" { + t.Fatalf("status %+v, want no applied generation with tier none", r.Status) + } +} + +func TestRefusedPrivilegeKeepsTierOfActiveExecBindings(t *testing.T) { + c, _, _, _ := setup(t) + withPrivilege := policy("policy", "ns") + withPrivilege.Spec.Privilege = privilegeSection(api.ActionDeny) + p := parseTypedPolicy(t, withPrivilege) + sections, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, nil, nil) // PrivilegeNotConfigured + c.backend = sections + if r := c.report(p, nil); r.Status.Tier != "none" || r.Status.Conditions[0].Reason != "PrivilegeNotConfigured" { + t.Fatalf("no active binding: %+v", r.Status) + } + b := binding() + c.active[bindingKey(b)] = installed{binding: b, plan: Plan{Policies: []CompiledPolicy{{Identity: PolicyIdentity{UID: p.identity.UID, Generation: 1}, Revision: "r"}}}} + if r := c.report(p, nil); r.Status.Tier != "bpf-lsm" || r.Status.Conditions[0].Reason != "PrivilegeNotConfigured" { + t.Fatalf("last-good exec binding: %+v", r.Status) + } +} From a591caa92e4600c5c31261b3c32c7367b78c7488 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:16:14 +0200 Subject: [PATCH 44/54] chore: pin storage to the node status API change Temporary replace to the fork commit of kubescape/storage#485, until it merges into the sandbox policy API branch. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- go.mod | 2 ++ go.sum | 2 ++ 2 files changed, 4 insertions(+) diff --git a/go.mod b/go.mod index ddd2494859..20709a35fc 100644 --- a/go.mod +++ b/go.mod @@ -517,3 +517,5 @@ require ( ) replace github.com/inspektor-gadget/inspektor-gadget => github.com/kubescape/inspektor-gadget v0.0.0-20261105131350-6d9170d1aca9 + +replace github.com/kubescape/storage => github.com/AlonLiwsky/storage v0.0.388-0.20261109102717-284ce4d29450 diff --git a/go.sum b/go.sum index e8aa7e6027..6aa0394798 100644 --- a/go.sum +++ b/go.sum @@ -215,6 +215,8 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8af github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20250520111509-a70c2aa677fa h1:x6kFzdPgBoLbyoNkA/jny0ENpoEz4wqY8lPTQL2DPkg= github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20250520111509-a70c2aa677fa/go.mod h1:gCLVsLfv1egrcZu+GoJATN5ts75F2s62ih/457eWzOw= +github.com/AlonLiwsky/storage v0.0.388-0.20261109102717-284ce4d29450 h1:EmcHBARi5DkGWNqZAGZWyhqS56RDzQktFIkFTSRq9wI= +github.com/AlonLiwsky/storage v0.0.388-0.20261109102717-284ce4d29450/go.mod h1:z3uiLdH1O3+eNiKuEOOalJy9uqH7ggNepaUSSWWm5ps= github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0/go.mod h1:XCW7KnZet0Opnr7HccfUw1PLc4CjHqpcaxW8DHklNkQ= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= From 9f99a2f9c43f1873126b46887ea1d40568cead62 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:47:11 +0200 Subject: [PATCH 45/54] fix(sandboxpolicy): count pod cgroup bindings in node status Privilege pod bindings are driven by the integration service, so the controller never saw them. When the container binding failed while a pod binding enforced, the report said no last-good enforcement exists and omitted appliedGeneration. The privilege backend now reports its bound pod plans, and the report counts them like container bindings. A pod whose plan changes only in generation records the new plan. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- docs/features/sandbox-policy.md | 5 ++- pkg/sandboxpolicy/igbackend/privilege.go | 23 +++++++++++- pkg/sandboxpolicy/igbackend/privilege_test.go | 37 +++++++++++++++++++ pkg/sandboxpolicy/sections.go | 10 +++++ pkg/sandboxpolicy/status.go | 29 +++++++++++---- pkg/sandboxpolicy/status_test.go | 28 ++++++++++++++ pkg/sandboxpolicy/types.go | 6 +++ 7 files changed, 127 insertions(+), 11 deletions(-) diff --git a/docs/features/sandbox-policy.md b/docs/features/sandbox-policy.md index 77aee6c42b..a8430c9a78 100644 --- a/docs/features/sandbox-policy.md +++ b/docs/features/sandbox-policy.md @@ -69,8 +69,9 @@ SQLite/files. Kubernetes owns the CRD objects. Per-policy/node status resources are output; the agent does not write parent aggregate status. Each report carries `appliedGeneration`, the oldest policy generation still -installed on the node. It stays on the last-good generation while a newer -revision is rejected, and is absent when no binding enforces the policy. At +installed on the node, counting container bindings and privilege pod cgroup +bindings. It stays on the last-good generation while a newer revision is +rejected, and is absent when no binding enforces the policy. At startup the agent reads the host's active LSM list once. Every section enforces through BPF LSM, so when `bpf` is missing every report says `tier: none`, and every enabled policy reports `Ready=False` with reason `BPFLSMInactive`. diff --git a/pkg/sandboxpolicy/igbackend/privilege.go b/pkg/sandboxpolicy/igbackend/privilege.go index dec4134eb2..cd30170949 100644 --- a/pkg/sandboxpolicy/igbackend/privilege.go +++ b/pkg/sandboxpolicy/igbackend/privilege.go @@ -378,7 +378,13 @@ func (b *PrivilegeBackend) BindPods(ctx context.Context, pods []sandboxpolicy.Po for uid, p := range wanted { combination, policies := Combination(p.Plan) current := b.pods[uid] - if len(policies) == 0 || current != nil && current.combination == combination { + if len(policies) == 0 { + continue + } + if current != nil && current.combination == combination { + // The same enforcement: keep the binding, record the newer plan + // (its generation) for status. + current.pod.Plan = p.Plan continue } slot, ok := b.slots[combination] @@ -395,6 +401,21 @@ func (b *PrivilegeBackend) BindPods(ctx context.Context, pods []sandboxpolicy.Po return errors.Join(append(errs, b.retire(ctx))...) } +// BoundPodPlans returns the plans of the pod cgroups bound now. An instance +// that is not ready protects nothing, so it reports none. +func (b *PrivilegeBackend) BoundPodPlans() []sandboxpolicy.Plan { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed || b.instance == nil || b.instance.Ready() != nil { + return nil + } + plans := make([]sandboxpolicy.Plan, 0, len(b.pods)) + for _, p := range b.pods { + plans = append(plans, p.pod.Plan) + } + return plans +} + // replaceInstance starts a candidate with every combination in use or prepared // (plus extra), binds every container (plus next, under nextKey) and, with // withPods, every pod cgroup, and only then stops the old instance. Without diff --git a/pkg/sandboxpolicy/igbackend/privilege_test.go b/pkg/sandboxpolicy/igbackend/privilege_test.go index 3b536054cd..05ed75b3df 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_test.go +++ b/pkg/sandboxpolicy/igbackend/privilege_test.go @@ -3,6 +3,7 @@ package igbackend import ( "context" "errors" + "fmt" "strings" "testing" @@ -474,3 +475,39 @@ func TestPrivilegeEmptyInstanceRetryWaits(t *testing.T) { t.Fatalf("prepare with a plan: %v instances=%d", err, len(e.instances)) } } + +func TestPrivilegeBoundPodPlansFollowBindings(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + gen := func(g int64) sandboxpolicy.Plan { + p := privPolicy("p1", api.ActionAllow, denyMount) + p.Identity.Generation = g + return privPlan(fmt.Sprintf("r%d", g), p) + } + if plans := b.BoundPodPlans(); len(plans) != 0 { + t.Fatalf("no instance: %+v", plans) + } + if err := b.Prepare(ctx, []sandboxpolicy.Plan{gen(1)}); err != nil { + t.Fatal(err) + } + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, gen(1))}); err != nil { + t.Fatal(err) + } + if plans := b.BoundPodPlans(); len(plans) != 1 || plans[0].Policies[0].Identity.Generation != 1 { + t.Fatalf("bound: %+v", plans) + } + // An edit with the same privilege content keeps the binding and reports the new generation. + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, gen(2))}); err != nil || len(e.instances) != 1 { + t.Fatalf("same content: %v instances %d", err, len(e.instances)) + } + if plans := b.BoundPodPlans(); len(plans) != 1 || plans[0].Policies[0].Identity.Generation != 2 { + t.Fatalf("generation not refreshed: %+v", plans) + } + if err := b.BindPods(ctx, nil); err != nil { + t.Fatal(err) + } + if plans := b.BoundPodPlans(); len(plans) != 0 { + t.Fatalf("unbound pod still reported: %+v", plans) + } +} diff --git a/pkg/sandboxpolicy/sections.go b/pkg/sandboxpolicy/sections.go index 2d7a547d76..68dc4f9c1c 100644 --- a/pkg/sandboxpolicy/sections.go +++ b/pkg/sandboxpolicy/sections.go @@ -89,6 +89,16 @@ func (s *SectionBackend) BindPods(ctx context.Context, pods []PodBinding) error return binder.BindPods(ctx, pods) } +// BoundPodPlans returns the plans bound to pod cgroups by a privilege backend +// that reports them. Pod bindings exist only for the privilege section. +func (s *SectionBackend) BoundPodPlans() []Plan { + reporter, ok := s.privilege.(PodPlanReporter) + if !ok { + return nil + } + return reporter.BoundPodPlans() +} + // Exec returns the exec section backend. func (s *SectionBackend) Exec() Backend { return s.exec } diff --git a/pkg/sandboxpolicy/status.go b/pkg/sandboxpolicy/status.go index a4f66f33e4..2196a66669 100644 --- a/pkg/sandboxpolicy/status.go +++ b/pkg/sandboxpolicy/status.go @@ -30,6 +30,17 @@ func (c *Controller) report(p desiredPolicy, failure error) *api.SandboxPolicyNo var cleanup *cleanupError cleanupPending := errors.As(failure, &cleanup) revisions := make(map[string]bool) + count := func(compiled CompiledPolicy) { + active++ + // The oldest generation still installed is the one this node can vouch for. + if status.AppliedGeneration == 0 || compiled.Identity.Generation < status.AppliedGeneration { + status.AppliedGeneration = compiled.Identity.Generation + } + revisions[fmt.Sprintf("generation=%d revision=%s", compiled.Identity.Generation, compiled.Revision)] = true + if compiled.Identity.Generation != p.identity.Generation { + stale = true + } + } for _, installed := range c.active { for _, compiled := range installed.plan.Policies { if compiled.Identity.UID != p.identity.UID { @@ -39,14 +50,16 @@ func (c *Controller) report(p desiredPolicy, failure error) *api.SandboxPolicyNo unconfirmed = true continue } - active++ - // The oldest generation still installed is the one this node can vouch for. - if status.AppliedGeneration == 0 || compiled.Identity.Generation < status.AppliedGeneration { - status.AppliedGeneration = compiled.Identity.Generation - } - revisions[fmt.Sprintf("generation=%d revision=%s", compiled.Identity.Generation, compiled.Revision)] = true - if compiled.Identity.Generation != p.identity.Generation { - stale = true + count(compiled) + } + } + // Pod cgroup bindings enforce too, even when no container binding does. + if reporter, ok := c.backend.(PodPlanReporter); ok { + for _, plan := range reporter.BoundPodPlans() { + for _, compiled := range plan.Policies { + if compiled.Identity.UID == p.identity.UID { + count(compiled) + } } } } diff --git a/pkg/sandboxpolicy/status_test.go b/pkg/sandboxpolicy/status_test.go index 0a78ec34aa..1b7013615a 100644 --- a/pkg/sandboxpolicy/status_test.go +++ b/pkg/sandboxpolicy/status_test.go @@ -115,3 +115,31 @@ func TestRefusedPrivilegeKeepsTierOfActiveExecBindings(t *testing.T) { t.Fatalf("last-good exec binding: %+v", r.Status) } } + +type podReportingBackend struct { + fakeBackend + pods []Plan +} + +func (b *podReportingBackend) BoundPodPlans() []Plan { return b.pods } + +// A pod cgroup binding enforces even when the container binding failed, so the +// report must not say that nothing enforces. +func TestPodBindingsCountAsEnforcement(t *testing.T) { + c, _, _, _ := setup(t) + p := parseTypedPolicy(t, policy("policy", "ns")) + p.identity.Generation = 3 + backend := &podReportingBackend{fakeBackend: fakeBackend{plans: map[string]Plan{}}} + backend.pods = []Plan{{Policies: []CompiledPolicy{ + {Identity: PolicyIdentity{UID: p.identity.UID, Generation: 2}, Revision: "r2"}, + {Identity: PolicyIdentity{UID: "other", Generation: 1}, Revision: "o"}, + }}} + c.backend = backend + r := c.report(p, errors.New("container binding refused")) + if r.Status.AppliedGeneration != 2 || r.Status.Tier != "bpf-lsm" || !r.Status.Stale { + t.Fatalf("status %+v", r.Status) + } + if strings.Contains(r.Status.Conditions[0].Message, "no last-good") { + t.Fatalf("pod binding ignored: %q", r.Status.Conditions[0].Message) + } +} diff --git a/pkg/sandboxpolicy/types.go b/pkg/sandboxpolicy/types.go index 21c292c1d9..fb5b58445c 100644 --- a/pkg/sandboxpolicy/types.go +++ b/pkg/sandboxpolicy/types.go @@ -131,6 +131,12 @@ type PodBinder interface { BindPods(context.Context, []PodBinding) error } +// PodPlanReporter optionally reports the plans that pod cgroup bindings +// enforce now, so node status counts them like container bindings. +type PodPlanReporter interface { + BoundPodPlans() []Plan +} + // SourceResolver reads ContainerProfile execs in the selected pod's namespace. // It must return an error for missing/incomplete profiles; empty successful results // mean a known complete profile with no execs. Rules retain their predicates. From 54f943c7f38675e04f1fd90c5d3039d89245ffff Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:08:16 +0200 Subject: [PATCH 46/54] fix(sandboxpolicy): node status stale, message and revision details Found on a kind cluster with the merged stack: - stale was set on any failure with installed bindings, even when they enforce the desired generation; now it follows the generation only. - without BPF LSM the Ready message named the first failing section instead of the node-wide cause. - pod plans carry no revision, so ActiveRevision listed "revision=". Co-Authored-By: Claude Opus 5.5 Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- docs/features/sandbox-policy.md | 5 ++++- pkg/sandboxpolicy/status.go | 16 +++++++++++--- pkg/sandboxpolicy/status_test.go | 36 ++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/docs/features/sandbox-policy.md b/docs/features/sandbox-policy.md index a8430c9a78..02c7f40b8b 100644 --- a/docs/features/sandbox-policy.md +++ b/docs/features/sandbox-policy.md @@ -71,7 +71,10 @@ are output; the agent does not write parent aggregate status. Each report carries `appliedGeneration`, the oldest policy generation still installed on the node, counting container bindings and privilege pod cgroup bindings. It stays on the last-good generation while a newer revision is -rejected, and is absent when no binding enforces the policy. At +rejected, and is absent when no binding enforces the policy. `stale` is set only +when an installed binding lags the desired generation or its protection cannot be +confirmed: a failure that leaves the current generation enforced is `Ready=False` +with `stale: false`. At startup the agent reads the host's active LSM list once. Every section enforces through BPF LSM, so when `bpf` is missing every report says `tier: none`, and every enabled policy reports `Ready=False` with reason `BPFLSMInactive`. diff --git a/pkg/sandboxpolicy/status.go b/pkg/sandboxpolicy/status.go index 2196a66669..127ca0c7f1 100644 --- a/pkg/sandboxpolicy/status.go +++ b/pkg/sandboxpolicy/status.go @@ -36,7 +36,11 @@ func (c *Controller) report(p desiredPolicy, failure error) *api.SandboxPolicyNo if status.AppliedGeneration == 0 || compiled.Identity.Generation < status.AppliedGeneration { status.AppliedGeneration = compiled.Identity.Generation } - revisions[fmt.Sprintf("generation=%d revision=%s", compiled.Identity.Generation, compiled.Revision)] = true + entry := fmt.Sprintf("generation=%d", compiled.Identity.Generation) + if compiled.Revision != "" { // pod plans carry no revision + entry += " revision=" + compiled.Revision + } + revisions[entry] = true if compiled.Identity.Generation != p.identity.Generation { stale = true } @@ -98,8 +102,11 @@ func (c *Controller) report(p desiredPolicy, failure error) *api.SandboxPolicyNo reason = "NotReady" message = "Desired revision rejected; no last-good enforcement exists" if active > 0 { + // Stale only when installed bindings lag the desired generation. message = "Desired revision rejected; retaining existing last-good bindings; new bindings may be unprotected" - stale = true + if !generationStale { + message = "Desired revision applied in part; installed bindings enforce it, but some bindings failed; see error" + } } if cleanupPending { reason = "CleanupPending" @@ -112,7 +119,10 @@ func (c *Controller) report(p desiredPolicy, failure error) *api.SandboxPolicyNo message = "Binding withdrawal committed; enforcement removed for that binding, resource cleanup pending" stale = generationStale } - if r, m, ok := unavailableReason(failure); ok { + // A node-wide cause names the node, not the first section it hit. + if r, m, ok := unavailableReason(c.options.EnforcementUnavailable); ok && p.spec.Mode != api.ModeOff { + reason, message = r, m + } else if r, m, ok := unavailableReason(failure); ok { reason, message = r, m } if unconfirmed { diff --git a/pkg/sandboxpolicy/status_test.go b/pkg/sandboxpolicy/status_test.go index 1b7013615a..08efd4cae3 100644 --- a/pkg/sandboxpolicy/status_test.go +++ b/pkg/sandboxpolicy/status_test.go @@ -143,3 +143,39 @@ func TestPodBindingsCountAsEnforcement(t *testing.T) { t.Fatalf("pod binding ignored: %q", r.Status.Conditions[0].Message) } } + +// Bindings on the desired generation are not stale, even when another binding +// failed; pod plans carry no revision, so ActiveRevision shows none. +func TestPartialFailureOnCurrentGenerationIsNotStale(t *testing.T) { + c, _, _, _ := setup(t) + p := parseTypedPolicy(t, policy("policy", "ns")) + p.identity.Generation = 3 + backend := &podReportingBackend{fakeBackend: fakeBackend{plans: map[string]Plan{}}} + backend.pods = []Plan{{Policies: []CompiledPolicy{{Identity: PolicyIdentity{UID: p.identity.UID, Generation: 3}}}}} + c.backend = backend + r := c.report(p, errors.New("exec binding refused")) + if r.Status.Stale || r.Status.AppliedGeneration != 3 || r.Status.Conditions[0].Status != metav1.ConditionFalse { + t.Fatalf("status %+v", r.Status) + } + if !strings.Contains(r.Status.Conditions[0].Message, "applied in part") { + t.Fatalf("message %q", r.Status.Conditions[0].Message) + } + if active := r.Status.Conditions[1]; active.Type != "ActiveRevision" || active.Message != "generation=3" { + t.Fatalf("active revision %+v", active) + } +} + +// Without BPF LSM the message names the node-wide cause, not the first +// section that also failed for it. +func TestNodeWideUnavailableMessageWins(t *testing.T) { + c, _, _, _ := setup(t) + withPrivilege := policy("policy", "ns") + withPrivilege.Spec.Privilege = privilegeSection(api.ActionDeny) + p := parseTypedPolicy(t, withPrivilege) + c.options.EnforcementUnavailable = &EnforcementUnavailableError{Section: "sandbox policy", Reason: "BPFLSMInactive", Detail: "BPF LSM is not active"} + section := &EnforcementUnavailableError{Section: "privilege", Reason: "BPFLSMInactive", Detail: "BPF LSM is not active"} + r := c.report(p, section) + if ready := r.Status.Conditions[0]; ready.Reason != "BPFLSMInactive" || !strings.HasPrefix(ready.Message, "sandbox policy enforcement") { + t.Fatalf("ready %+v", ready) + } +} From 2f1b68ee046831b7f96f4200c4dd40f14e1ce834 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:02:42 +0200 Subject: [PATCH 47/54] chore: pin storage to the merged node status API change kubescape/storage#485 merged, so drop the temporary replace to the fork. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- go.mod | 4 +--- go.sum | 6 ++++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 20709a35fc..d06087b9d4 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/kubescape/backend v0.0.39 github.com/kubescape/go-logger v0.0.35 github.com/kubescape/k8s-interface v0.0.214 - github.com/kubescape/storage v0.0.388-0.20261105083812-136b4c98606f + github.com/kubescape/storage v0.0.388-0.20261109155453-fa55b25a70a3 github.com/kubescape/workerpool v0.0.0-20250526074519-0e4a4e7f44cf github.com/moby/sys/mountinfo v0.7.2 github.com/opcoder0/fanotify v0.4.2 @@ -517,5 +517,3 @@ require ( ) replace github.com/inspektor-gadget/inspektor-gadget => github.com/kubescape/inspektor-gadget v0.0.0-20261105131350-6d9170d1aca9 - -replace github.com/kubescape/storage => github.com/AlonLiwsky/storage v0.0.388-0.20261109102717-284ce4d29450 diff --git a/go.sum b/go.sum index 6aa0394798..fb5471c377 100644 --- a/go.sum +++ b/go.sum @@ -215,8 +215,6 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8af github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20250520111509-a70c2aa677fa h1:x6kFzdPgBoLbyoNkA/jny0ENpoEz4wqY8lPTQL2DPkg= github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20250520111509-a70c2aa677fa/go.mod h1:gCLVsLfv1egrcZu+GoJATN5ts75F2s62ih/457eWzOw= -github.com/AlonLiwsky/storage v0.0.388-0.20261109102717-284ce4d29450 h1:EmcHBARi5DkGWNqZAGZWyhqS56RDzQktFIkFTSRq9wI= -github.com/AlonLiwsky/storage v0.0.388-0.20261109102717-284ce4d29450/go.mod h1:z3uiLdH1O3+eNiKuEOOalJy9uqH7ggNepaUSSWWm5ps= github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0/go.mod h1:XCW7KnZet0Opnr7HccfUw1PLc4CjHqpcaxW8DHklNkQ= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= @@ -1461,6 +1459,8 @@ github.com/kubescape/storage v0.0.352-0.20261013133511-eb6406e01cb9 h1:pPFehIHOZ github.com/kubescape/storage v0.0.352-0.20261013133511-eb6406e01cb9/go.mod h1:OSYXj9TzVAuuz+Gn2bIzp3Vg8rofTPH7x5Asek0N7mE= github.com/kubescape/storage v0.0.388-0.20261105083812-136b4c98606f h1:KTA6ctZBLIi+XRZwKFtl8etQZOTPciSy/nfAtBizzlc= github.com/kubescape/storage v0.0.388-0.20261105083812-136b4c98606f/go.mod h1:z3uiLdH1O3+eNiKuEOOalJy9uqH7ggNepaUSSWWm5ps= +github.com/kubescape/storage v0.0.388-0.20261109155453-fa55b25a70a3 h1:oSyi5Tiixb+25/HMG03DiKtmiQiyQbuXs0p1F/WKfik= +github.com/kubescape/storage v0.0.388-0.20261109155453-fa55b25a70a3/go.mod h1:z3uiLdH1O3+eNiKuEOOalJy9uqH7ggNepaUSSWWm5ps= github.com/kubescape/syft v1.32.0-ks.2 h1:xdUksUmKEyyVKsTfJDYW8Z5HawVJtelsUolPOsWtDx0= github.com/kubescape/syft v1.32.0-ks.2/go.mod h1:E6Kd4iBM2ljUOUQvSt7hVK6vBwaHkMXwcvBZmGMSY5o= github.com/kubescape/workerpool v0.0.0-20250526074519-0e4a4e7f44cf h1:hI0jVwrB6fT4GJWvuUjzObfci1CUknrZdRHfnRVtKM0= @@ -1865,6 +1865,7 @@ github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8r github.com/sebdah/goldie/v2 v2.8.0/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= github.com/seccomp/libseccomp-golang v0.11.0 h1:SDkcBRqGLP+sezmMACkxO1EfgbghxIxnRKfd6mHUEis= github.com/seccomp/libseccomp-golang v0.11.0/go.mod h1:5m1Lk8E9OwgZTTVz4bBOer7JuazaBa+xTkM895tDiWc= +github.com/seccomp/libseccomp-golang v0.11.1 h1:wuk4ZjSx6kyQII4rj6G6fvVzRHQaSiPvccJazDagu4g= github.com/seccomp/libseccomp-golang v0.11.1/go.mod h1:5m1Lk8E9OwgZTTVz4bBOer7JuazaBa+xTkM895tDiWc= github.com/secure-systems-lab/go-securesystemslib v0.10.0 h1:l+H5ErcW0PAehBNrBxoGv1jjNpGYdZ9RcheFkB2WI14= github.com/secure-systems-lab/go-securesystemslib v0.10.0/go.mod h1:MRKONWmRoFzPNQ9USRF9i1mc7MvAVvF1LlW8X5VWDvk= @@ -2919,6 +2920,7 @@ k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kms v0.35.0/go.mod h1:VT+4ekZAdrZDMgShK37vvlyHUVhwI9t/9tvh0AyCWmQ= k8s.io/kms v0.37.0 h1:WDW3pSc5yFB1Od4YXXL3i5oK+vXI2v++LAV82+Ks1Z0= k8s.io/kms v0.37.0/go.mod h1:0U6zfwLkfJr9O/jLgz2GARnKLh2uIhkq2EeaU66J5ow= +k8s.io/kms v0.37.1 h1:MD1kA8AdFF/N5sGP8gnvL6vTo/oeaQXbmLjWmmJHLc8= k8s.io/kms v0.37.1/go.mod h1:0U6zfwLkfJr9O/jLgz2GARnKLh2uIhkq2EeaU66J5ow= k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= From 3544f776d1c3f7c98201600417670d1b1d340e03 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:50:48 +0200 Subject: [PATCH 48/54] fix(sandboxpolicy): count a pod binding only where witness counters confirm it Node status counted a pod plan as enforcing on instance readiness alone. Now a pod plan counts only when the pod was bound in the current instance before any of its containers existed, and its slot is healthy. A pod plan whose slot has a fault makes the policy report ProtectionUnconfirmed. A pod gets a new pod binding only while none of its containers exists (the container collection learns a container at create time and drops pause containers). A running pod relies on its container bindings, which arm the running tasks. A pod binding carried into a new instance is not confirmed once the pod's containers started, because the new instance arms none of their tasks. Pod bindings are subtree bindings that admit the runtime helper exemption. A Strict slot's health also fails on a node tracking fault. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/igbackend/privilege.go | 62 +++++++++++++------ .../igbackend/privilege_native.go | 16 ++++- pkg/sandboxpolicy/igbackend/privilege_test.go | 61 ++++++++++++++++-- pkg/sandboxpolicy/integration/service.go | 17 +++-- pkg/sandboxpolicy/sections.go | 4 +- pkg/sandboxpolicy/status.go | 13 +++- pkg/sandboxpolicy/status_test.go | 19 +++++- pkg/sandboxpolicy/types.go | 16 ++++- 8 files changed, 170 insertions(+), 38 deletions(-) diff --git a/pkg/sandboxpolicy/igbackend/privilege.go b/pkg/sandboxpolicy/igbackend/privilege.go index cd30170949..a34d684866 100644 --- a/pkg/sandboxpolicy/igbackend/privilege.go +++ b/pkg/sandboxpolicy/igbackend/privilege.go @@ -74,17 +74,24 @@ func (p *privilegeBinding) close() error { type podBinding struct { pod sandboxpolicy.PodBinding combination string + // confirmed: bound in the current instance before any container of the + // pod started, so every task of the pod's containers is judged from its + // first exec. + confirmed bool } // podRuntime is the runtime name of a pod cgroup identity in the bindings // controller. No container runtime has this name. const podRuntime = "kubernetes-pod" -// nativePod returns the bindings controller identity of a pod cgroup. Tasks -// already in it are not armed: runc init of a container being created can be -// there. +// nativePod returns the bindings controller identity of a pod cgroup. It +// covers the container cgroups below it (Subtree) and admits the runtime +// helper exemption for the OCI hooks that run before a container's workload +// (RuntimeHelpers). Tasks already in it are not armed: runc init of a +// container being created can be there. func nativePod(p sandboxpolicy.PodBinding, slot uint32) enforcement.Binding { - return enforcement.Binding{Container: enforcement.ContainerIdentity{Runtime: podRuntime, ID: string(p.PodUID), CgroupID: p.CgroupID, CgroupPath: p.CgroupPath}, Slot: slot, SkipArming: true} + return enforcement.Binding{Container: enforcement.ContainerIdentity{Runtime: podRuntime, ID: string(p.PodUID), CgroupID: p.CgroupID, CgroupPath: p.CgroupPath}, Slot: slot, + Subtree: true, RuntimeHelpers: true, SkipArming: true} } // PrivilegeBackend owns ONE enforce_privilege instance per node. @@ -382,9 +389,18 @@ func (b *PrivilegeBackend) BindPods(ctx context.Context, pods []sandboxpolicy.Po continue } if current != nil && current.combination == combination { - // The same enforcement: keep the binding, record the newer plan - // (its generation) for status. - current.pod.Plan = p.Plan + // The same enforcement: keep the binding and its confirmation, + // record the newer pod state (the plan's generation) for status. + current.pod = p + continue + } + // A pod binding arms no running task: a pod whose containers + // started relies on its container bindings, and a pod already bound + // keeps its last-good binding. + if p.ContainersStarted { + if current != nil { + current.pod.ContainersStarted = true + } continue } slot, ok := b.slots[combination] @@ -395,25 +411,31 @@ func (b *PrivilegeBackend) BindPods(ctx context.Context, pods []sandboxpolicy.Po errs = append(errs, err) continue } - b.pods[uid] = &podBinding{pod: p, combination: combination} + b.pods[uid] = &podBinding{pod: p, combination: combination, confirmed: true} logger.L().Info("sandbox privilege pod bind", helpers.String("namespace", p.Namespace), helpers.String("pod", p.PodName)) } return errors.Join(append(errs, b.retire(ctx))...) } -// BoundPodPlans returns the plans of the pod cgroups bound now. An instance -// that is not ready protects nothing, so it reports none. -func (b *PrivilegeBackend) BoundPodPlans() []sandboxpolicy.Plan { +// BoundPodPlans returns the plans of the pod cgroups bound now (see +// sandboxpolicy.PodPlanReporter). An instance that is not ready protects +// nothing, so it reports none. +func (b *PrivilegeBackend) BoundPodPlans() (confirmed, faulted []sandboxpolicy.Plan) { b.mu.Lock() defer b.mu.Unlock() if b.closed || b.instance == nil || b.instance.Ready() != nil { - return nil + return nil, nil } - plans := make([]sandboxpolicy.Plan, 0, len(b.pods)) for _, p := range b.pods { - plans = append(plans, p.pod.Plan) + slot, ok := b.slots[p.combination] + switch { + case !ok || b.instance.SlotHealth(slot) != nil: + faulted = append(faulted, p.pod.Plan) + case p.confirmed: + confirmed = append(confirmed, p.pod.Plan) + } } - return plans + return confirmed, faulted } // replaceInstance starts a candidate with every combination in use or prepared @@ -422,9 +444,13 @@ func (b *PrivilegeBackend) BoundPodPlans() []sandboxpolicy.Plan { // withPods, the pod bindings are dropped. On failure the old instance and // state stay. func (b *PrivilegeBackend) replaceInstance(ctx context.Context, prepared, extra map[string][]sandboxpolicy.CompiledPolicy, nextKey identity, next *privilegeBinding, withPods bool) error { - pods := b.pods - if !withPods { - pods = map[types.UID]*podBinding{} + // A new instance arms no task under a pod binding, so a carried pod + // binding is confirmed only while no container of the pod has started. + pods := map[types.UID]*podBinding{} + if withPods { + for uid, p := range b.pods { + pods[uid] = &podBinding{pod: p.pod, combination: p.combination, confirmed: !p.pod.ContainersStarted} + } } bindings := map[identity]*privilegeBinding{} for k, v := range b.bindings { diff --git a/pkg/sandboxpolicy/igbackend/privilege_native.go b/pkg/sandboxpolicy/igbackend/privilege_native.go index 4ee6e33952..cc00246be9 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_native.go +++ b/pkg/sandboxpolicy/igbackend/privilege_native.go @@ -48,6 +48,7 @@ type nativePrivilegeInstance struct { id string bindings *enforcement.Bindings revision uint64 + strict func(slot uint32) bool } func (i *nativePrivilegeInstance) Ready() error { @@ -58,6 +59,9 @@ func (i *nativePrivilegeInstance) Ready() error { return nil } +// SlotHealth fails on a fault counted in the slot, and, for a Strict slot, on +// a node fault: a child of an unconfirmed task can start in any cgroup +// (common/arming.h). An unreadable counter is a fault too. func (i *nativePrivilegeInstance) SlotHealth(slot uint32) error { n, err := i.bindings.SlotFaults(slot) if err != nil { @@ -66,6 +70,16 @@ func (i *nativePrivilegeInstance) SlotHealth(slot uint32) error { if n > 0 { return fmt.Errorf("%d tracking faults in privilege slot %d", n, slot) } + if i.strict == nil || !i.strict(slot) { + return nil + } + w, err := i.bindings.ArmingWitness() + if err != nil { + return err + } + if w.NodeFaults > 0 { + return fmt.Errorf("%d node tracking faults affect Strict privilege slot %d", w.NodeFaults, slot) + } return nil } @@ -93,7 +107,7 @@ func (e *nativePrivilegeEngine) Start(ctx context.Context, id string, config pri if err := handler.Init(params); err != nil { return nil, err } - instance := &nativePrivilegeInstance{manager: e.manager, id: id, revision: config.Policy.Generation} + instance := &nativePrivilegeInstance{manager: e.manager, id: id, revision: config.Policy.Generation, strict: privruntime.StrictSlots(config.Policy)} names := config.Policy.RuleNames events := simple.New("sandbox-privilege-verdict", simple.WithPriority(100), simple.OnInit(func(gc operators.GadgetContext) error { ds := gc.GetDataSources()["verdict"] diff --git a/pkg/sandboxpolicy/igbackend/privilege_test.go b/pkg/sandboxpolicy/igbackend/privilege_test.go index 05ed75b3df..a183b9eb34 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_test.go +++ b/pkg/sandboxpolicy/igbackend/privilege_test.go @@ -318,7 +318,7 @@ func TestPrivilegePodBindings(t *testing.T) { carried := false for _, initial := range e.configs[1].InitialBindings { if initial.Container.CgroupID == 100 { - carried = initial.SkipArming && initial.Container.Runtime == podRuntime && initial.Container.ID == "a" + carried = initial.SkipArming && initial.Subtree && initial.RuntimeHelpers && initial.Container.Runtime == podRuntime && initial.Container.ID == "a" } } if !carried { @@ -485,7 +485,7 @@ func TestPrivilegeBoundPodPlansFollowBindings(t *testing.T) { p.Identity.Generation = g return privPlan(fmt.Sprintf("r%d", g), p) } - if plans := b.BoundPodPlans(); len(plans) != 0 { + if plans, faulted := b.BoundPodPlans(); len(plans)+len(faulted) != 0 { t.Fatalf("no instance: %+v", plans) } if err := b.Prepare(ctx, []sandboxpolicy.Plan{gen(1)}); err != nil { @@ -494,20 +494,71 @@ func TestPrivilegeBoundPodPlansFollowBindings(t *testing.T) { if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, gen(1))}); err != nil { t.Fatal(err) } - if plans := b.BoundPodPlans(); len(plans) != 1 || plans[0].Policies[0].Identity.Generation != 1 { + if plans, _ := b.BoundPodPlans(); len(plans) != 1 || plans[0].Policies[0].Identity.Generation != 1 { t.Fatalf("bound: %+v", plans) } // An edit with the same privilege content keeps the binding and reports the new generation. if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, gen(2))}); err != nil || len(e.instances) != 1 { t.Fatalf("same content: %v instances %d", err, len(e.instances)) } - if plans := b.BoundPodPlans(); len(plans) != 1 || plans[0].Policies[0].Identity.Generation != 2 { + if plans, _ := b.BoundPodPlans(); len(plans) != 1 || plans[0].Policies[0].Identity.Generation != 2 { t.Fatalf("generation not refreshed: %+v", plans) } if err := b.BindPods(ctx, nil); err != nil { t.Fatal(err) } - if plans := b.BoundPodPlans(); len(plans) != 0 { + if plans, faulted := b.BoundPodPlans(); len(plans)+len(faulted) != 0 { t.Fatalf("unbound pod still reported: %+v", plans) } } + +// A pod plan counts only where the witness counters prove it: the pod was +// bound in the current instance before its containers started, and its slot +// has no fault. +func TestPrivilegePodPlansNeedWitness(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + p1 := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + p2 := privPlan("r2", privPolicy("p2", api.ActionDeny)) + if err := b.Prepare(ctx, []sandboxpolicy.Plan{p1}); err != nil { + t.Fatal(err) + } + started := testPodBinding("s", 200, p1) + started.ContainersStarted = true + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, p1), started}); err != nil { + t.Fatal(err) + } + if _, ok := e.instances[0].bound[200]; ok { + t.Fatal("a pod whose containers started must not get a pod binding") + } + if plans, faulted := b.BoundPodPlans(); len(plans) != 1 || len(faulted) != 0 { + t.Fatalf("bound before start: %d confirmed, %d faulted", len(plans), len(faulted)) + } + // A slot fault: the pod plan is faulted, not confirmed. + e.instances[0].faults = map[uint32]uint64{e.instances[0].bound[100]: 1} + if plans, faulted := b.BoundPodPlans(); len(plans) != 0 || len(faulted) != 1 { + t.Fatalf("slot fault: %d confirmed, %d faulted", len(plans), len(faulted)) + } + e.instances[0].faults = nil + // The pod's containers start, then a new instance carries the pod + // binding. The new instance arms none of the pod's running tasks, so the + // pod plan is no longer confirmed. + running := testPodBinding("a", 100, p1) + running.ContainersStarted = true + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{running}); err != nil { + t.Fatal(err) + } + if plans, _ := b.BoundPodPlans(); len(plans) != 1 { + t.Fatal("a pod bound before its containers started stays confirmed in its instance") + } + if err := b.Prepare(ctx, []sandboxpolicy.Plan{p1, p2}); err != nil || len(e.instances) != 2 { + t.Fatalf("prepare p2: %v", err) + } + if _, ok := e.instances[1].bound[100]; !ok { + t.Fatal("the pod binding must be carried") + } + if plans, faulted := b.BoundPodPlans(); len(plans) != 0 || len(faulted) != 0 { + t.Fatalf("carried after start: %d confirmed, %d faulted", len(plans), len(faulted)) + } +} diff --git a/pkg/sandboxpolicy/integration/service.go b/pkg/sandboxpolicy/integration/service.go index d3806d0839..e7e33fe9b5 100644 --- a/pkg/sandboxpolicy/integration/service.go +++ b/pkg/sandboxpolicy/integration/service.go @@ -447,7 +447,7 @@ func (s *Service) bindPodPlans(ctx context.Context, scheduled []sandboxpolicy.Bi firstSeen := map[types.UID]time.Time{} polling := false if len(plans) > 0 { - layout, known, containerCgroups := s.podCgroupLayout() + layout, known, containerCgroups, started := s.podCgroupLayout() for _, c := range scheduled { plan, ok := plans[c.PodUID] if !ok { @@ -473,7 +473,7 @@ func (s *Service) bindPodPlans(ctx context.Context, scheduled []sandboxpolicy.Bi if containerCgroups[id] { continue } - pods = append(pods, sandboxpolicy.PodBinding{PodUID: c.PodUID, Namespace: c.Namespace, PodName: c.PodName, CgroupID: id, CgroupPath: path, Plan: plan}) + pods = append(pods, sandboxpolicy.PodBinding{PodUID: c.PodUID, Namespace: c.Namespace, PodName: c.PodName, CgroupID: id, CgroupPath: path, Plan: plan, ContainersStarted: started[c.PodUID]}) } } s.podFirstSeen, s.podPolling = firstSeen, polling @@ -513,23 +513,30 @@ func (s *Service) scheduledPods(ctx context.Context) ([]sandboxpolicy.Binding, m // podCgroupLayout reads the pod cgroup layout from a running pod container, // and returns the cgroup IDs of the running containers. -func (s *Service) podCgroupLayout() (podCgroupLayout, bool, map[uint64]bool) { +func (s *Service) podCgroupLayout() (podCgroupLayout, bool, map[uint64]bool, map[types.UID]bool) { var layout podCgroupLayout found := false cgroups := map[uint64]bool{} + // The pods with a container that exists: the collection learns a + // container when the runtime creates it, before its workload runs, and + // it drops pause containers. + started := map[types.UID]bool{} if s.options.Containers == nil { - return layout, false, cgroups + return layout, false, cgroups, started } s.options.Containers.ContainerRange(func(c *containercollection.Container) { if c == nil { return } cgroups[c.CgroupID] = true + if c.K8s.PodUID != "" { + started[types.UID(c.K8s.PodUID)] = true + } if !found { layout, found = learnPodCgroupLayout(c.CgroupV2, types.UID(c.K8s.PodUID)) } }) - return layout, found, cgroups + return layout, found, cgroups, started } // ContainerCallback is deliberately nonblocking: event delivery must never hold diff --git a/pkg/sandboxpolicy/sections.go b/pkg/sandboxpolicy/sections.go index 68dc4f9c1c..2d6eedca91 100644 --- a/pkg/sandboxpolicy/sections.go +++ b/pkg/sandboxpolicy/sections.go @@ -91,10 +91,10 @@ func (s *SectionBackend) BindPods(ctx context.Context, pods []PodBinding) error // BoundPodPlans returns the plans bound to pod cgroups by a privilege backend // that reports them. Pod bindings exist only for the privilege section. -func (s *SectionBackend) BoundPodPlans() []Plan { +func (s *SectionBackend) BoundPodPlans() (confirmed, faulted []Plan) { reporter, ok := s.privilege.(PodPlanReporter) if !ok { - return nil + return nil, nil } return reporter.BoundPodPlans() } diff --git a/pkg/sandboxpolicy/status.go b/pkg/sandboxpolicy/status.go index 127ca0c7f1..f785a68ec0 100644 --- a/pkg/sandboxpolicy/status.go +++ b/pkg/sandboxpolicy/status.go @@ -57,15 +57,24 @@ func (c *Controller) report(p desiredPolicy, failure error) *api.SandboxPolicyNo count(compiled) } } - // Pod cgroup bindings enforce too, even when no container binding does. + // Pod cgroup bindings enforce too, even when no container binding does, + // but only where the witness counters confirm it. if reporter, ok := c.backend.(PodPlanReporter); ok { - for _, plan := range reporter.BoundPodPlans() { + confirmed, faulted := reporter.BoundPodPlans() + for _, plan := range confirmed { for _, compiled := range plan.Policies { if compiled.Identity.UID == p.identity.UID { count(compiled) } } } + for _, plan := range faulted { + for _, compiled := range plan.Policies { + if compiled.Identity.UID == p.identity.UID { + unconfirmed = true + } + } + } } if active > 0 { status.Tier = "bpf-lsm" diff --git a/pkg/sandboxpolicy/status_test.go b/pkg/sandboxpolicy/status_test.go index 08efd4cae3..7fce5dc2f6 100644 --- a/pkg/sandboxpolicy/status_test.go +++ b/pkg/sandboxpolicy/status_test.go @@ -118,10 +118,25 @@ func TestRefusedPrivilegeKeepsTierOfActiveExecBindings(t *testing.T) { type podReportingBackend struct { fakeBackend - pods []Plan + pods, faulted []Plan } -func (b *podReportingBackend) BoundPodPlans() []Plan { return b.pods } +func (b *podReportingBackend) BoundPodPlans() (confirmed, faulted []Plan) { return b.pods, b.faulted } + +// A pod binding whose slot has a fault cannot be confirmed: it never counts as +// enforcement, and the policy reports that protection is unconfirmed. +func TestFaultedPodBindingIsUnconfirmed(t *testing.T) { + c, _, _, _ := setup(t) + p := parseTypedPolicy(t, policy("policy", "ns")) + p.identity.Generation = 3 + backend := &podReportingBackend{fakeBackend: fakeBackend{plans: map[string]Plan{}}} + backend.faulted = []Plan{{Policies: []CompiledPolicy{{Identity: PolicyIdentity{UID: p.identity.UID, Generation: 3}}}}} + c.backend = backend + r := c.report(p, nil) + if r.Status.AppliedGeneration != 0 || r.Status.Conditions[0].Status != metav1.ConditionFalse || r.Status.Conditions[0].Reason != "ProtectionUnconfirmed" { + t.Fatalf("status %+v", r.Status) + } +} // A pod cgroup binding enforces even when the container binding failed, so the // report must not say that nothing enforces. diff --git a/pkg/sandboxpolicy/types.go b/pkg/sandboxpolicy/types.go index fb5b58445c..b78ee42c7c 100644 --- a/pkg/sandboxpolicy/types.go +++ b/pkg/sandboxpolicy/types.go @@ -121,6 +121,12 @@ type PodBinding struct { CgroupID uint64 CgroupPath string Plan Plan + // ContainersStarted: a container of the pod exists already (the + // container collection knows it). A pod binding does not arm the tasks + // already in the pod cgroup, so such a pod gets no new pod binding, and a + // pod binding carried into a new instance is not confirmed. Its + // containers rely on their own container bindings. + ContainersStarted bool } // PodBinder optionally binds pod cgroups. BindPods receives the complete set: @@ -131,10 +137,14 @@ type PodBinder interface { BindPods(context.Context, []PodBinding) error } -// PodPlanReporter optionally reports the plans that pod cgroup bindings -// enforce now, so node status counts them like container bindings. +// PodPlanReporter optionally reports the plans of the pod cgroup bindings, so +// node status counts them like container bindings. confirmed are the plans +// that the witness counters prove: the pod was bound before its containers +// started and its slot has no fault. faulted are the plans whose slot has a +// fault, so their protection cannot be confirmed. A pod bound after its +// containers started is in neither: its containers rely on their own bindings. type PodPlanReporter interface { - BoundPodPlans() []Plan + BoundPodPlans() (confirmed, faulted []Plan) } // SourceResolver reads ContainerProfile execs in the selected pod's namespace. From 10d7a2629cb888fe4be2594708c2f27813e76c78 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:52:37 +0200 Subject: [PATCH 49/54] fix(sandboxpolicy): prepare the privilege slot before exec commits SectionBackend now commits in two phases. The privilege backend first prepares the plan's slot (PrepareReplace), which can fail and binds nothing. Only then exec commits, and the privilege bind is a live map write. A failed prepare leaves both sections on their last-good plan with no exec rollback. The rollback stays only for a failed live bind. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/igbackend/privilege.go | 25 +++++++++++ pkg/sandboxpolicy/igbackend/privilege_test.go | 25 +++++++++++ pkg/sandboxpolicy/sections.go | 11 ++++- pkg/sandboxpolicy/sections_test.go | 41 +++++++++++++++++++ pkg/sandboxpolicy/types.go | 8 ++++ 5 files changed, 109 insertions(+), 1 deletion(-) diff --git a/pkg/sandboxpolicy/igbackend/privilege.go b/pkg/sandboxpolicy/igbackend/privilege.go index a34d684866..ffd65e0339 100644 --- a/pkg/sandboxpolicy/igbackend/privilege.go +++ b/pkg/sandboxpolicy/igbackend/privilege.go @@ -135,6 +135,7 @@ const healRetry = 30 * time.Second var ( _ sandboxpolicy.Backend = (*PrivilegeBackend)(nil) + _ sandboxpolicy.ReplacePreparer = (*PrivilegeBackend)(nil) _ sandboxpolicy.ProtectionAttester = (*PrivilegeBackend)(nil) _ sandboxpolicy.PodBinder = (*PrivilegeBackend)(nil) ) @@ -270,6 +271,30 @@ func (b *PrivilegeBackend) Replace(ctx context.Context, binding sandboxpolicy.Bi return commit() } +// PrepareReplace makes sure the plan's combination has a slot in a ready +// instance, replacing the node instance if needed, and binds nothing. The +// Replace that follows is then a live map write. +func (b *PrivilegeBackend) PrepareReplace(ctx context.Context, _ sandboxpolicy.Binding, plan sandboxpolicy.Plan) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return fmt.Errorf("privilege backend is closed") + } + combination, policies := Combination(plan) + if len(policies) == 0 { + return fmt.Errorf("plan has no privilege section") + } + if _, ok := b.slots[combination]; ok && b.instance != nil && b.instance.Ready() == nil { + return nil + } + extra := map[string][]sandboxpolicy.CompiledPolicy{combination: policies} + err := b.replaceInstance(ctx, b.prepared, extra, identity{}, nil, true) + if err != nil && len(b.prepared) > 0 { + err = b.replaceInstance(ctx, nil, extra, identity{}, nil, true) + } + return errors.Join(err, b.retire(ctx)) +} + func (b *PrivilegeBackend) Withdraw(ctx context.Context, binding sandboxpolicy.Binding) (sandboxpolicy.WithdrawResult, error) { b.mu.Lock() defer b.mu.Unlock() diff --git a/pkg/sandboxpolicy/igbackend/privilege_test.go b/pkg/sandboxpolicy/igbackend/privilege_test.go index a183b9eb34..2502b56bcc 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_test.go +++ b/pkg/sandboxpolicy/igbackend/privilege_test.go @@ -562,3 +562,28 @@ func TestPrivilegePodPlansNeedWitness(t *testing.T) { t.Fatalf("carried after start: %d confirmed, %d faulted", len(plans), len(faulted)) } } + +// PrepareReplace creates the slot and binds nothing; the Replace that follows +// is a live map write in the same instance. +func TestPrivilegePrepareReplaceMakesBindLive(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + p1 := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + if err := b.PrepareReplace(ctx, privBinding(1), p1); err != nil || len(e.instances) != 1 { + t.Fatalf("prepare: %v instances %d", err, len(e.instances)) + } + if _, bound := e.instances[0].bound[1]; bound { + t.Fatal("prepare must not bind") + } + if r, err := b.Replace(ctx, privBinding(1), p1); err != nil || !r.Committed || len(e.instances) != 1 { + t.Fatalf("replace: %+v %v instances %d", r, err, len(e.instances)) + } + if b.Stats().Prepared != 1 || b.Stats().Replaced != 0 { + t.Fatalf("the bind must be a live write: %+v", b.Stats()) + } + // A prepared combination needs no new instance. + if err := b.PrepareReplace(ctx, privBinding(2), p1); err != nil || len(e.instances) != 1 { + t.Fatalf("second prepare: %v instances %d", err, len(e.instances)) + } +} diff --git a/pkg/sandboxpolicy/sections.go b/pkg/sandboxpolicy/sections.go index 2d6eedca91..0a882cbe0b 100644 --- a/pkg/sandboxpolicy/sections.go +++ b/pkg/sandboxpolicy/sections.go @@ -219,6 +219,14 @@ func (s *SectionBackend) Replace(ctx context.Context, binding Binding, plan Plan return ReplaceResult{}, s.unavailable } + // Two phases: first prepare the privilege slot, which can fail and + // changes no binding; then commit exec, then make the live privilege + // bind. A failed prepare leaves both sections on their last-good plan. + if preparer, ok := s.privilege.(ReplacePreparer); ok && want.privilege.Revision != "" && want.privilege.Revision != old.privilege.Revision { + if err := preparer.PrepareReplace(ctx, binding, want.privilege); err != nil { + return ReplaceResult{}, fmt.Errorf("privilege section: %w", err) + } + } e := apply(ctx, s.exec, binding, old.exec, want.exec) if !e.committed { return ReplaceResult{ProtectionUnconfirmed: e.unconfirmed}, fmt.Errorf("exec section: %w", e.err) @@ -229,7 +237,8 @@ func (s *SectionBackend) Replace(ctx context.Context, binding Binding, plan Plan } p := apply(ctx, s.privilege, binding, oldPrivilege, want.privilege) if !p.committed { - // Return exec to its previous plan. + // The live bind failed after a successful prepare (identity check + // or map write): return exec to its previous plan. back := apply(ctx, s.exec, binding, want.exec, old.exec) return ReplaceResult{ProtectionUnconfirmed: e.unconfirmed || p.unconfirmed || back.unconfirmed || !back.committed}, errors.Join(fmt.Errorf("privilege section: %w", p.err), back.err) diff --git a/pkg/sandboxpolicy/sections_test.go b/pkg/sandboxpolicy/sections_test.go index 0d4d702297..d60f6cdf79 100644 --- a/pkg/sandboxpolicy/sections_test.go +++ b/pkg/sandboxpolicy/sections_test.go @@ -410,3 +410,44 @@ func TestNodeWithBPFLSMReportsTierBPFLSM(t *testing.T) { t.Fatalf("report %+v", r) } } + +type replacePreparingBackend struct { + fakeBackend + prepareErr error + prepared []string +} + +func (b *replacePreparingBackend) PrepareReplace(_ context.Context, _ Binding, plan Plan) error { + b.prepared = append(b.prepared, plan.Revision) + return b.prepareErr +} + +// The privilege slot is prepared before exec commits: a failed prepare leaves +// both sections on their last-good plan, with no exec rollback. +func TestSectionBackendPreparesPrivilegeBeforeExec(t *testing.T) { + exec := &fakeBackend{plans: map[string]Plan{}} + privilege := &replacePreparingBackend{fakeBackend: fakeBackend{plans: map[string]Plan{}}, prepareErr: errors.New("slot does not fit")} + s, _ := NewSectionBackend(exec, privilege, nil) + b := Binding{ContainerID: "c"} + execPolicy := CompiledPolicy{Revision: "e", Exec: api.ExecSection{Default: api.ActionDeny}} + privPolicy := CompiledPolicy{Revision: "p", Privilege: privilegeSection(api.ActionAllow)} + r, err := s.Replace(context.Background(), b, sectionPlan("r1", execPolicy, privPolicy)) + if err == nil || r.Committed || r.ProtectionUnconfirmed { + t.Fatalf("failed prepare: %+v %v", r, err) + } + if exec.replacements+exec.withdrawals != 0 || privilege.replacements != 0 { + t.Fatalf("a failed prepare must change no section: exec %d/%d privilege %d", exec.replacements, exec.withdrawals, privilege.replacements) + } + privilege.prepareErr = nil + if r, err := s.Replace(context.Background(), b, sectionPlan("r1", execPolicy, privPolicy)); err != nil || !r.Committed { + t.Fatalf("replace: %+v %v", r, err) + } + // An unchanged privilege revision needs no prepare. + execPolicy.Revision = "e2" + if _, err := s.Replace(context.Background(), b, sectionPlan("r2", execPolicy, privPolicy)); err != nil { + t.Fatal(err) + } + if len(privilege.prepared) != 2 { + t.Fatalf("prepares %v", privilege.prepared) + } +} diff --git a/pkg/sandboxpolicy/types.go b/pkg/sandboxpolicy/types.go index b78ee42c7c..aa05404067 100644 --- a/pkg/sandboxpolicy/types.go +++ b/pkg/sandboxpolicy/types.go @@ -103,6 +103,14 @@ type ProtectionAttester interface { Attest(context.Context, Binding, string) error } +// ReplacePreparer optionally does the part of a Replace that can fail without +// changing any binding (for example a new policy slot in a node instance), so +// that a Replace with the same plan right after it is a live map write. +// SectionBackend prepares the privilege section before it commits exec. +type ReplacePreparer interface { + PrepareReplace(ctx context.Context, binding Binding, plan Plan) error +} + // SlotPreparer optionally prepares enforcement for pods scheduled to this node // before their containers start. Each plan is what a container of such a pod // will get. It is a hint: a later Replace must still work without it. From e5d42e22544959fb8d21566509eb2fe45f3b3c8a Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:55:01 +0200 Subject: [PATCH 50/54] fix(sandboxpolicy): privilege reports Ready only on a qualified kernel The privilege section keeps enforcing on any kernel with BPF LSM, but a privilege policy reports Ready=False with reason PrivilegeKernelUnqualified unless the kernel release is qualified. sandboxPolicyQualifiedKernels sets the list; unset uses the AL2023 amd64 fleet kernels 6.12.113-138.209 and 6.18.56-125.164, and ["*"] admits every kernel (test clusters). Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- cmd/main.go | 4 +- docs/features/sandbox-policy.md | 21 +++++++++++ pkg/config/config.go | 3 ++ pkg/sandboxpolicy/igbackend/kernel.go | 44 ++++++++++++++++++++++ pkg/sandboxpolicy/igbackend/kernel_test.go | 25 ++++++++++++ pkg/sandboxpolicy/integration/service.go | 16 +++++++- pkg/sandboxpolicy/sections.go | 18 ++++++++- pkg/sandboxpolicy/status_test.go | 18 +++++++++ 8 files changed, 144 insertions(+), 5 deletions(-) create mode 100644 pkg/sandboxpolicy/igbackend/kernel.go create mode 100644 pkg/sandboxpolicy/igbackend/kernel_test.go diff --git a/cmd/main.go b/cmd/main.go index 9af62780a3..5c56c22c19 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -577,8 +577,8 @@ func main() { Kubernetes: k8sClient.GetKubernetesClient(), Dynamic: k8sClient.GetDynamicClient(), Containers: mainHandler.GetContainerCollection(), Tracers: mainHandler.GetTracerCollection(), NodeName: cfg.NodeName, AgentNamespace: cfg.NamespaceName, ImageName: cfg.SandboxPolicyImage, ImageArchive: "tracers.tar", - PrivilegeImageName: cfg.SandboxPolicyPrivilegeImage, - ContainerdSocket: cfg.SandboxPolicyContainerdSocket, ControlRoot: cfg.SandboxPolicyControlRoot, MaxOwners: cfg.SandboxPolicyMaxOwners, + PrivilegeImageName: cfg.SandboxPolicyPrivilegeImage, QualifiedKernels: cfg.SandboxPolicyQualifiedKernels, + ContainerdSocket: cfg.SandboxPolicyContainerdSocket, ControlRoot: cfg.SandboxPolicyControlRoot, MaxOwners: cfg.SandboxPolicyMaxOwners, Sources: sandboxintegration.NewProfileResolver(storageClient.GetStorageClient()), Receivers: thirdPartyTracers.ThirdPartyEventReceivers, OnError: func(err error) { logger.L().Ctx(ctx).Error("sandbox policy service diagnostic", helpers.Error(err)) }, diff --git a/docs/features/sandbox-policy.md b/docs/features/sandbox-policy.md index 02c7f40b8b..ea28d687bb 100644 --- a/docs/features/sandbox-policy.md +++ b/docs/features/sandbox-policy.md @@ -79,6 +79,27 @@ startup the agent reads the host's active LSM list once. Every section enforces through BPF LSM, so when `bpf` is missing every report says `tier: none`, and every enabled policy reports `Ready=False` with reason `BPFLSMInactive`. +A privilege pod cgroup binding counts only where the witness counters confirm +it: the pod was bound before any of its containers existed, in the instance +that enforces now, and its slot has no tracking fault. A pod whose containers +already exist gets no pod binding; its containers rely on their own bindings. +A pod binding whose slot has a fault makes the policy report +`ProtectionUnconfirmed`. For a Strict slot a node tracking fault (a lost child +mark of a `CLONE_INTO_CGROUP` fork anywhere on the node) is a fault too; a new +instance clears it. + +The privilege section reports Ready only on a qualified kernel: +`sandboxPolicyQualifiedKernels` lists kernel releases (`uname -r`); unset uses +the AL2023 amd64 fleet kernels `6.12.113-138.209.amzn2023.x86_64` and +`6.18.56-125.164.amzn2023.x86_64`, and `["*"]` admits every kernel. On another +kernel the section still enforces, but every privilege policy reports +`Ready=False` with reason `PrivilegeKernelUnqualified`. + +A privilege change commits in two phases: the privilege slot is prepared first +(this can replace the node instance and binds nothing), then exec commits, then +the privilege bind is a live map write. A failed prepare leaves both sections on +their last-good plan. + ## Reconciliation and failure behavior Configuration watch events are **refresh hints**, not trusted queued payloads. diff --git a/pkg/config/config.go b/pkg/config/config.go index e03c52567c..20812aed9f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -123,6 +123,7 @@ type Config struct { EnableSandboxPolicy bool `mapstructure:"sandboxPolicyEnabled"` SandboxPolicyImage string `mapstructure:"sandboxPolicyImage"` SandboxPolicyPrivilegeImage string `mapstructure:"sandboxPolicyPrivilegeImage"` + SandboxPolicyQualifiedKernels []string `mapstructure:"sandboxPolicyQualifiedKernels"` SandboxPolicyContainerdSocket string `mapstructure:"sandboxPolicyContainerdSocket"` SandboxPolicyControlRoot string `mapstructure:"sandboxPolicyControlRoot"` SandboxPolicyMaxOwners uint64 `mapstructure:"sandboxPolicyMaxOwners"` @@ -228,6 +229,8 @@ func LoadConfigOptional(path string, errNotFound bool) (Config, error) { viper.SetDefault("sandboxPolicyEnabled", false) viper.SetDefault("sandboxPolicyImage", "") viper.SetDefault("sandboxPolicyPrivilegeImage", "") + // Unset: the built-in qualified kernel list. ["*"] admits every kernel. + viper.SetDefault("sandboxPolicyQualifiedKernels", nil) viper.SetDefault("sandboxPolicyContainerdSocket", "") viper.SetDefault("sandboxPolicyControlRoot", "") viper.SetDefault("sandboxPolicyMaxOwners", 4) diff --git a/pkg/sandboxpolicy/igbackend/kernel.go b/pkg/sandboxpolicy/igbackend/kernel.go new file mode 100644 index 0000000000..b5d5ef5a59 --- /dev/null +++ b/pkg/sandboxpolicy/igbackend/kernel.go @@ -0,0 +1,44 @@ +package igbackend + +import ( + "fmt" + "runtime" + "slices" + + "github.com/kubescape/node-agent/pkg/sandboxpolicy" + "golang.org/x/sys/unix" +) + +// QualifiedKernels are the kernel releases (uname -r) on which the privilege +// section passed its kernel qualification: the AL2023 amd64 fleet kernels. +var QualifiedKernels = []string{ + "6.12.113-138.209.amzn2023.x86_64", + "6.18.56-125.164.amzn2023.x86_64", +} + +// AnyKernel in a qualified list admits every kernel (test clusters). +const AnyKernel = "*" + +// CheckQualifiedKernel returns an EnforcementUnavailableError with reason +// PrivilegeKernelUnqualified unless release is in qualified (nil: +// QualifiedKernels). The privilege section still enforces on such a kernel, +// but never reports Ready. +func CheckQualifiedKernel(release string, qualified []string) error { + if qualified == nil { + qualified = QualifiedKernels + } + if slices.Contains(qualified, AnyKernel) || slices.Contains(qualified, release) { + return nil + } + return &sandboxpolicy.EnforcementUnavailableError{Section: "privilege", Reason: "PrivilegeKernelUnqualified", + Detail: fmt.Sprintf("kernel %s (%s) is not qualified; qualified kernels: %v", release, runtime.GOARCH, qualified)} +} + +// KernelRelease returns the running kernel release (uname -r). +func KernelRelease() (string, error) { + var u unix.Utsname + if err := unix.Uname(&u); err != nil { + return "", err + } + return unix.ByteSliceToString(u.Release[:]), nil +} diff --git a/pkg/sandboxpolicy/igbackend/kernel_test.go b/pkg/sandboxpolicy/igbackend/kernel_test.go new file mode 100644 index 0000000000..987e3c70c1 --- /dev/null +++ b/pkg/sandboxpolicy/igbackend/kernel_test.go @@ -0,0 +1,25 @@ +package igbackend + +import ( + "errors" + "testing" + + "github.com/kubescape/node-agent/pkg/sandboxpolicy" +) + +func TestCheckQualifiedKernel(t *testing.T) { + if err := CheckQualifiedKernel("6.12.113-138.209.amzn2023.x86_64", nil); err != nil { + t.Fatalf("a fleet kernel is qualified: %v", err) + } + err := CheckQualifiedKernel("6.12.114-139.1.amzn2023.x86_64", nil) + var u *sandboxpolicy.EnforcementUnavailableError + if !errors.As(err, &u) || u.Reason != "PrivilegeKernelUnqualified" { + t.Fatalf("another kernel is not qualified: %v", err) + } + if err := CheckQualifiedKernel("6.17.0-lab", []string{AnyKernel}); err != nil { + t.Fatalf("the wildcard admits every kernel: %v", err) + } + if err := CheckQualifiedKernel("6.17.0-lab", []string{}); err == nil { + t.Fatal("an empty list admits no kernel") + } +} diff --git a/pkg/sandboxpolicy/integration/service.go b/pkg/sandboxpolicy/integration/service.go index e7e33fe9b5..46fddf79a5 100644 --- a/pkg/sandboxpolicy/integration/service.go +++ b/pkg/sandboxpolicy/integration/service.go @@ -53,6 +53,11 @@ type ServiceOptions struct { OnError func(error) // PrivilegeImageName enables the privilege section. Empty rejects it. PrivilegeImageName string + // QualifiedKernels lists the kernel releases (uname -r) on which the + // privilege section may report Ready; nil uses igbackend.QualifiedKernels, + // and "*" admits every kernel. On another kernel it enforces but stays + // NotReady (PrivilegeKernelUnqualified). + QualifiedKernels []string } type serviceBackend interface { @@ -192,12 +197,21 @@ func NewService(ctx context.Context, options ServiceOptions) (*Service, error) { return nil, err } } - s.backend, err = sandboxpolicy.NewSectionBackend(full, privilege, privilegeUnavailable) + sections, err := sandboxpolicy.NewSectionBackend(full, privilege, privilegeUnavailable) if err != nil { _ = full.Close(ctx) _ = s.closeRuntime() return nil, err } + if privilege != nil { + release, rerr := igbackend.KernelRelease() + if rerr != nil { + sections.SetPrivilegeUnqualified(&sandboxpolicy.EnforcementUnavailableError{Section: "privilege", Reason: "PrivilegeKernelUnqualified", Detail: fmt.Sprintf("kernel release: %v", rerr)}) + } else { + sections.SetPrivilegeUnqualified(igbackend.CheckQualifiedKernel(release, options.QualifiedKernels)) + } + } + s.backend = sections s.Controller, err = sandboxpolicy.NewController(options.Dynamic, s.backend, options.Sources, readinessWriter{inner: &sandboxpolicy.DynamicStatusWriter{Client: options.Dynamic}, observed: &s.inputEpoch, snapshot: &s.snapshotEpoch}, sandboxpolicy.Options{Node: api.NodeReference{Name: node.Name, UID: node.UID}, AgentNamespace: options.AgentNamespace, EnforcementUnavailable: lsmUnavailable}) if err != nil { _ = s.backend.Close(ctx) diff --git a/pkg/sandboxpolicy/sections.go b/pkg/sandboxpolicy/sections.go index 0a882cbe0b..7f59d03e2f 100644 --- a/pkg/sandboxpolicy/sections.go +++ b/pkg/sandboxpolicy/sections.go @@ -31,6 +31,9 @@ type SectionBackend struct { privilege Backend // nil: privilege sections are rejected with unavailable // unavailable says why privilege sections are rejected (privilege == nil). unavailable error + // unqualified says why the privilege section, which enforces, must not + // report Ready (for example a kernel that is not qualified). + unqualified error active map[string]sectionPlans } @@ -62,8 +65,19 @@ func NewSectionBackend(exec, privilege Backend, unavailable error) (*SectionBack return &SectionBackend{exec: exec, privilege: privilege, unavailable: unavailable, active: map[string]sectionPlans{}}, nil } -// PrivilegeUnavailable returns why privilege sections cannot be enforced. -func (s *SectionBackend) PrivilegeUnavailable() error { return s.unavailable } +// PrivilegeUnavailable returns why privilege sections cannot be enforced, or +// why they enforce but must not report Ready. +func (s *SectionBackend) PrivilegeUnavailable() error { + if s.unavailable != nil { + return s.unavailable + } + return s.unqualified +} + +// SetPrivilegeUnqualified keeps the privilege section enforcing, but its +// policies never report Ready while err is set. Call it before the +// controller starts. +func (s *SectionBackend) SetPrivilegeUnqualified(err error) { s.unqualified = err } // execOnly reports whether the plans need only the exec backend. func (p sectionPlans) execOnly() bool { return p.privilege.Revision == "" && !p.privilegeCleanup } diff --git a/pkg/sandboxpolicy/status_test.go b/pkg/sandboxpolicy/status_test.go index 7fce5dc2f6..fca791d66d 100644 --- a/pkg/sandboxpolicy/status_test.go +++ b/pkg/sandboxpolicy/status_test.go @@ -194,3 +194,21 @@ func TestNodeWideUnavailableMessageWins(t *testing.T) { t.Fatalf("ready %+v", ready) } } + +// On a kernel that is not qualified the privilege section enforces, but a +// privilege policy never reports Ready. +func TestUnqualifiedKernelIsNotReady(t *testing.T) { + c, _, _, _ := setup(t) + withPrivilege := policy("policy", "ns") + withPrivilege.Spec.Privilege = privilegeSection(api.ActionDeny) + p := parseTypedPolicy(t, withPrivilege) + sections, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, &fakeBackend{plans: map[string]Plan{}}, nil) + sections.SetPrivilegeUnqualified(&EnforcementUnavailableError{Section: "privilege", Reason: "PrivilegeKernelUnqualified", Detail: "kernel 6.17 is not qualified"}) + c.backend = sections + if r := c.report(p, nil); r.Status.Conditions[0].Status != metav1.ConditionFalse || r.Status.Conditions[0].Reason != "PrivilegeKernelUnqualified" { + t.Fatalf("unqualified kernel: %+v", r.Status.Conditions[0]) + } + if _, err := sections.Replace(context.Background(), Binding{ContainerID: "c"}, sectionPlan("r", CompiledPolicy{Revision: "p", Privilege: privilegeSection(api.ActionDeny)})); err != nil { + t.Fatalf("the privilege section still enforces: %v", err) + } +} From cc3cef5ff45535fb17f03afa2d83af59c9c0359b Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:55:56 +0200 Subject: [PATCH 51/54] fix(sandboxpolicy): keep a privilege withdraw pending while the instance is not ready An instance that is not ready may still run its programs. A withdraw there no longer drops the record: it stays pending and reports the withdrawal as not committed, so the controller retries it. A new instance never carries a pending withdraw, and the old instance stops after the switch. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/igbackend/privilege.go | 30 +++++++++++++++---- pkg/sandboxpolicy/igbackend/privilege_test.go | 30 +++++++++++++++---- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/pkg/sandboxpolicy/igbackend/privilege.go b/pkg/sandboxpolicy/igbackend/privilege.go index ffd65e0339..2d71a0c74e 100644 --- a/pkg/sandboxpolicy/igbackend/privilege.go +++ b/pkg/sandboxpolicy/igbackend/privilege.go @@ -54,6 +54,10 @@ type privilegeBinding struct { // lease pins the live container identity while it is bound (nil when the // backend has no validator). lease Lease + // pendingUnbind: withdrawn while the instance was not ready. Its + // programs may still enforce, so the record stays until the unbind + // succeeds or a new instance (which never binds it) replaces the old one. + pendingUnbind bool } func (p *privilegeBinding) check() error { @@ -206,7 +210,7 @@ func (b *PrivilegeBackend) Replace(ctx context.Context, binding sandboxpolicy.Bi if b.instance != nil && health == nil { health = b.instance.Ready() } - if current != nil && current.revision == plan.Revision && current.combination == combination && health == nil { + if current != nil && current.revision == plan.Revision && current.combination == combination && health == nil && !current.pendingUnbind { result.Committed = true return result, b.retire(ctx) } @@ -303,10 +307,15 @@ func (b *PrivilegeBackend) Withdraw(ctx context.Context, binding sandboxpolicy.B if current == nil { return sandboxpolicy.WithdrawResult{Committed: true}, b.retire(ctx) } - // An instance that is not ready protects nothing, so the removal is - // already true; only a ready instance must drop the map entry first. - if b.instance != nil && b.instance.Ready() == nil { + // An instance that is not ready may still run its programs: keep the + // record as pending and let the caller retry. + if b.instance != nil { + if err := b.instance.Ready(); err != nil { + current.pendingUnbind = true + return sandboxpolicy.WithdrawResult{ProtectionUnconfirmed: true}, fmt.Errorf("privilege unbind pending: %w", err) + } if err := b.instance.Unbind(native(binding)); err != nil { + current.pendingUnbind = true return sandboxpolicy.WithdrawResult{ProtectionUnconfirmed: true}, err } } @@ -477,8 +486,15 @@ func (b *PrivilegeBackend) replaceInstance(ctx context.Context, prepared, extra pods[uid] = &podBinding{pod: p.pod, combination: p.combination, confirmed: !p.pod.ContainersStarted} } } + // A pending unbind is not carried: the new instance never binds it, and + // the old instance stops after the switch. bindings := map[identity]*privilegeBinding{} + var withdrawn []*privilegeBinding for k, v := range b.bindings { + if v.pendingUnbind { + withdrawn = append(withdrawn, v) + continue + } bindings[k] = v } if next != nil { @@ -546,7 +562,11 @@ func (b *PrivilegeBackend) replaceInstance(ctx context.Context, prepared, extra } b.instance, b.slots, b.combinations, b.bindings, b.prepared, b.pods = candidate, slots, wanted, bindings, prepared, pods b.stats.Replacements++ - return nil + var errs []error + for _, v := range withdrawn { + errs = append(errs, v.close()) + } + return errors.Join(errs...) } // retire stops replaced instances and keeps the ones that still fail. diff --git a/pkg/sandboxpolicy/igbackend/privilege_test.go b/pkg/sandboxpolicy/igbackend/privilege_test.go index 2502b56bcc..5c3a2657b4 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_test.go +++ b/pkg/sandboxpolicy/igbackend/privilege_test.go @@ -193,25 +193,45 @@ func TestPrivilegeRetriesFailedStop(t *testing.T) { } } -func TestPrivilegeWithdrawCommitsWhenInstanceDown(t *testing.T) { +// An instance that is not ready may still run its programs, so a withdraw +// there stays pending: it is retried, and a new instance never carries it. +func TestPrivilegeWithdrawPendingWhenInstanceNotReady(t *testing.T) { e := &fakePrivilegeEngine{} b, _ := NewPrivilege(e, nil, "n") ctx := context.Background() if _, err := b.Replace(ctx, privBinding(1), privPlan("r1", privPolicy("p1", api.ActionAllow))); err != nil { t.Fatal(err) } - e.instances[0].readyErr = errors.New("stopped") - if r, err := b.Withdraw(ctx, privBinding(1)); err != nil || !r.Committed { - t.Fatalf("withdraw on a stopped instance must commit: %+v %v", r, err) + e.instances[0].readyErr = errors.New("not ready") + if r, err := b.Withdraw(ctx, privBinding(1)); err == nil || r.Committed || !r.ProtectionUnconfirmed { + t.Fatalf("withdraw on an instance that is not ready must stay pending: %+v %v", r, err) } - // The next container recovers the node with a new instance. + // The retry on a ready instance unbinds and commits. e.instances[0].readyErr = nil + if r, err := b.Withdraw(ctx, privBinding(1)); err != nil || !r.Committed { + t.Fatalf("retry: %+v %v", r, err) + } + if _, bound := e.instances[0].bound[1]; bound { + t.Fatal("the retry must unbind") + } + // Pending again, then a new instance replaces the old one: the withdrawn + // container is not carried, and the next retry commits. + if _, err := b.Replace(ctx, privBinding(1), privPlan("r1", privPolicy("p1", api.ActionAllow))); err != nil { + t.Fatal(err) + } + e.instances[0].readyErr = errors.New("not ready") + if r, _ := b.Withdraw(ctx, privBinding(1)); r.Committed { + t.Fatal("must stay pending") + } if r, err := b.Replace(ctx, privBinding(2), privPlan("r2", privPolicy("p1", api.ActionAllow))); err != nil || !r.Committed { t.Fatalf("recovery: %+v %v", r, err) } if c := e.configs[len(e.configs)-1]; len(c.InitialBindings) > 1 { t.Fatalf("the withdrawn container must not be carried: %+v", c.InitialBindings) } + if r, err := b.Withdraw(ctx, privBinding(1)); err != nil || !r.Committed { + t.Fatalf("after the replacement: %+v %v", r, err) + } } func TestPrivilegeOversizedPrepareDoesNotBlockContainers(t *testing.T) { From b73a4bff5a59f414f2c03d0b932e5a500efea713 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:06:44 +0200 Subject: [PATCH 52/54] fix(sandboxpolicy): review follow-ups on pod confirmation, pending withdraws and heals - A pod's containers count as started from the API status and from container cgroups under the pod cgroup, not only from the container collection's pod UID. - While the instance is not ready, every pod plan is faulted. - BindPods keeps a pod's cgroup identity when the plan is unchanged, and moves a started pod to its new slot (unconfirmed) when the plan changes. - A pending record replaced under the same key never closes the lease the new record reuses, and a pending record never takes the heal path. - Prepare replaces an instance that is not ready (rate limited), so a pending withdraw cannot stay forever. - PrepareReplace reserves its slot until the next Replace. - Node faults alone heal at most every 10 minutes. - Removing the privilege section withdraws it before exec commits. - The unqualified-kernel status says the section enforces; an empty kernel list is the built-in list. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- docs/features/sandbox-policy.md | 18 ++-- pkg/sandboxpolicy/availability.go | 6 ++ pkg/sandboxpolicy/igbackend/kernel.go | 8 +- pkg/sandboxpolicy/igbackend/kernel_test.go | 8 +- pkg/sandboxpolicy/igbackend/privilege.go | 82 +++++++++++++------ .../igbackend/privilege_native.go | 8 +- pkg/sandboxpolicy/igbackend/privilege_test.go | 67 +++++++++++++++ pkg/sandboxpolicy/integration/bindings.go | 33 ++++++-- .../integration/bindings_test.go | 36 ++++++++ pkg/sandboxpolicy/integration/service.go | 75 +++++++++++------ pkg/sandboxpolicy/sections.go | 28 +++++-- pkg/sandboxpolicy/sections_test.go | 45 ++++++++++ 12 files changed, 339 insertions(+), 75 deletions(-) diff --git a/docs/features/sandbox-policy.md b/docs/features/sandbox-policy.md index ea28d687bb..ddcd686af8 100644 --- a/docs/features/sandbox-policy.md +++ b/docs/features/sandbox-policy.md @@ -81,15 +81,21 @@ every enabled policy reports `Ready=False` with reason `BPFLSMInactive`. A privilege pod cgroup binding counts only where the witness counters confirm it: the pod was bound before any of its containers existed, in the instance -that enforces now, and its slot has no tracking fault. A pod whose containers -already exist gets no pod binding; its containers rely on their own bindings. +that enforces now, and its slot has no tracking fault. "Its containers exist" +comes from the pod's API status and from the container collection (by pod UID +and by cgroup path); a container created after that snapshot is seen at the +next call. A pod whose containers already exist gets no pod binding; its +containers rely on their own bindings. A bound pod whose plan changes later +moves to its new slot, unconfirmed. While the instance is not ready, every pod +binding is unconfirmed. A pod binding whose slot has a fault makes the policy report `ProtectionUnconfirmed`. For a Strict slot a node tracking fault (a lost child mark of a `CLONE_INTO_CGROUP` fork anywhere on the node) is a fault too; a new -instance clears it. +instance clears it; node faults alone start a new instance at most every 10 +minutes, because their cause may persist. The privilege section reports Ready only on a qualified kernel: -`sandboxPolicyQualifiedKernels` lists kernel releases (`uname -r`); unset uses +`sandboxPolicyQualifiedKernels` lists kernel releases (`uname -r`); unset or empty uses the AL2023 amd64 fleet kernels `6.12.113-138.209.amzn2023.x86_64` and `6.18.56-125.164.amzn2023.x86_64`, and `["*"]` admits every kernel. On another kernel the section still enforces, but every privilege policy reports @@ -98,7 +104,9 @@ kernel the section still enforces, but every privilege policy reports A privilege change commits in two phases: the privilege slot is prepared first (this can replace the node instance and binds nothing), then exec commits, then the privilege bind is a live map write. A failed prepare leaves both sections on -their last-good plan. +their last-good plan. Removing the privilege section withdraws it first: a +withdraw on an instance that is not ready stays pending and is retried, and exec +does not change meanwhile. ## Reconciliation and failure behavior diff --git a/pkg/sandboxpolicy/availability.go b/pkg/sandboxpolicy/availability.go index 073ac7f979..80a77779b3 100644 --- a/pkg/sandboxpolicy/availability.go +++ b/pkg/sandboxpolicy/availability.go @@ -9,9 +9,15 @@ import ( // node. Reason becomes the Ready condition reason in the node status. type EnforcementUnavailableError struct { Section, Reason, Detail string + // Enforcing: the section enforces, but must not report Ready (for + // example on a kernel that is not qualified). + Enforcing bool } func (e *EnforcementUnavailableError) Error() string { + if e.Enforcing { + return fmt.Sprintf("%s enforces on this node but is not qualified: %s", e.Section, e.Detail) + } return fmt.Sprintf("%s enforcement is unavailable on this node: %s", e.Section, e.Detail) } diff --git a/pkg/sandboxpolicy/igbackend/kernel.go b/pkg/sandboxpolicy/igbackend/kernel.go index b5d5ef5a59..944f97f4b9 100644 --- a/pkg/sandboxpolicy/igbackend/kernel.go +++ b/pkg/sandboxpolicy/igbackend/kernel.go @@ -20,18 +20,18 @@ var QualifiedKernels = []string{ const AnyKernel = "*" // CheckQualifiedKernel returns an EnforcementUnavailableError with reason -// PrivilegeKernelUnqualified unless release is in qualified (nil: +// PrivilegeKernelUnqualified unless release is in qualified (empty: // QualifiedKernels). The privilege section still enforces on such a kernel, // but never reports Ready. func CheckQualifiedKernel(release string, qualified []string) error { - if qualified == nil { + if len(qualified) == 0 { qualified = QualifiedKernels } if slices.Contains(qualified, AnyKernel) || slices.Contains(qualified, release) { return nil } - return &sandboxpolicy.EnforcementUnavailableError{Section: "privilege", Reason: "PrivilegeKernelUnqualified", - Detail: fmt.Sprintf("kernel %s (%s) is not qualified; qualified kernels: %v", release, runtime.GOARCH, qualified)} + return &sandboxpolicy.EnforcementUnavailableError{Section: "privilege", Reason: "PrivilegeKernelUnqualified", Enforcing: true, + Detail: fmt.Sprintf("kernel %s (%s) is not in the qualified list %v", release, runtime.GOARCH, qualified)} } // KernelRelease returns the running kernel release (uname -r). diff --git a/pkg/sandboxpolicy/igbackend/kernel_test.go b/pkg/sandboxpolicy/igbackend/kernel_test.go index 987e3c70c1..fcae5153e7 100644 --- a/pkg/sandboxpolicy/igbackend/kernel_test.go +++ b/pkg/sandboxpolicy/igbackend/kernel_test.go @@ -2,6 +2,7 @@ package igbackend import ( "errors" + "strings" "testing" "github.com/kubescape/node-agent/pkg/sandboxpolicy" @@ -19,7 +20,10 @@ func TestCheckQualifiedKernel(t *testing.T) { if err := CheckQualifiedKernel("6.17.0-lab", []string{AnyKernel}); err != nil { t.Fatalf("the wildcard admits every kernel: %v", err) } - if err := CheckQualifiedKernel("6.17.0-lab", []string{}); err == nil { - t.Fatal("an empty list admits no kernel") + if err := CheckQualifiedKernel("6.12.113-138.209.amzn2023.x86_64", []string{}); err != nil { + t.Fatalf("an empty list (a Helm default) is the built-in list: %v", err) + } + if !u.Enforcing || !strings.Contains(u.Error(), "enforces") { + t.Fatalf("the section still enforces: %q", u.Error()) } } diff --git a/pkg/sandboxpolicy/igbackend/privilege.go b/pkg/sandboxpolicy/igbackend/privilege.go index 2d71a0c74e..1f33d45ba4 100644 --- a/pkg/sandboxpolicy/igbackend/privilege.go +++ b/pkg/sandboxpolicy/igbackend/privilege.go @@ -119,9 +119,12 @@ type PrivilegeBackend struct { bindings map[identity]*privilegeBinding pods map[types.UID]*podBinding prepared map[string][]sandboxpolicy.CompiledPolicy - retiring []PrivilegeInstance - stats PrivilegeStats - closed bool + // reserved is the combination of the last PrepareReplace, kept in every + // replacement until the Replace that follows it consumes it. + reserved map[string][]sandboxpolicy.CompiledPolicy + retiring []PrivilegeInstance + stats PrivilegeStats + closed bool // emptyFailed is when starting an instance with no slot last failed. emptyFailed time.Time // lastHeal is when a tracking fault last started a new instance. @@ -137,6 +140,15 @@ const emptyRetry = 30 * time.Second // every reconcile would add load then. The binding stays unconfirmed meanwhile. const healRetry = 30 * time.Second +// nodeHealRetry replaces healRetry when only node faults fail the health: a +// new instance cannot fix a cause that persists, and each reload unconfirms +// the pod bindings of running pods. +const nodeHealRetry = 10 * time.Minute + +// ErrNodeFaultsOnly marks a slot health error caused only by node tracking +// faults (see PrivilegeInstance.SlotHealth). +var ErrNodeFaultsOnly = errors.New("node tracking faults") + var ( _ sandboxpolicy.Backend = (*PrivilegeBackend)(nil) _ sandboxpolicy.ReplacePreparer = (*PrivilegeBackend)(nil) @@ -195,6 +207,7 @@ func (b *PrivilegeBackend) Replace(ctx context.Context, binding sandboxpolicy.Bi current := b.bindings[k] var result sandboxpolicy.ReplaceResult var health error + defer func() { b.reserved = nil }() // consumed, whatever the outcome if current != nil { result.ActiveRevision = current.revision health = b.health(current) @@ -214,10 +227,16 @@ func (b *PrivilegeBackend) Replace(ctx context.Context, binding sandboxpolicy.Bi result.Committed = true return result, b.retire(ctx) } - if current != nil && current.revision == plan.Revision && current.combination == combination && + if current != nil && current.revision == plan.Revision && current.combination == combination && !current.pendingUnbind && b.instance.Ready() == nil && current.check() == nil { - // Only tracking faults: heal with a new instance, at most every healRetry. - if time.Since(b.lastHeal) < healRetry { + // Only tracking faults: heal with a new instance, at most every + // healRetry. Node faults alone come from any workload on the node + // and may persist, so they heal at most every nodeHealRetry. + retry := healRetry + if errors.Is(health, ErrNodeFaultsOnly) { + retry = nodeHealRetry + } + if time.Since(b.lastHeal) < retry { return result, health } b.lastHeal = time.Now() @@ -276,8 +295,9 @@ func (b *PrivilegeBackend) Replace(ctx context.Context, binding sandboxpolicy.Bi } // PrepareReplace makes sure the plan's combination has a slot in a ready -// instance, replacing the node instance if needed, and binds nothing. The -// Replace that follows is then a live map write. +// instance, replacing the node instance if needed, and binds nothing. The slot +// stays reserved in later replacements until a Replace consumes it, so the +// Replace that follows is normally a live map write. func (b *PrivilegeBackend) PrepareReplace(ctx context.Context, _ sandboxpolicy.Binding, plan sandboxpolicy.Plan) error { b.mu.Lock() defer b.mu.Unlock() @@ -288,6 +308,7 @@ func (b *PrivilegeBackend) PrepareReplace(ctx context.Context, _ sandboxpolicy.B if len(policies) == 0 { return fmt.Errorf("plan has no privilege section") } + b.reserved = map[string][]sandboxpolicy.CompiledPolicy{combination: policies} if _, ok := b.slots[combination]; ok && b.instance != nil && b.instance.Ready() == nil { return nil } @@ -360,6 +381,12 @@ func (b *PrivilegeBackend) Prepare(ctx context.Context, plans []sandboxpolicy.Pl // is slow, so the first container of a privilege policy would find no // instance ready (for example the first pod after a node-agent restart). missing := b.instance == nil && time.Since(b.emptyFailed) >= emptyRetry + // An instance that is not ready protects nothing that can be confirmed, + // and it holds pending withdrawals: replace it, at most every healRetry. + if b.instance != nil && b.instance.Ready() != nil && time.Since(b.lastHeal) >= healRetry { + b.lastHeal = time.Now() + missing = true + } for _, plan := range plans { combination, policies := Combination(plan) if len(policies) == 0 { @@ -423,18 +450,19 @@ func (b *PrivilegeBackend) BindPods(ctx context.Context, pods []sandboxpolicy.Po continue } if current != nil && current.combination == combination { - // The same enforcement: keep the binding and its confirmation, - // record the newer pod state (the plan's generation) for status. - current.pod = p + // The same enforcement: keep the binding, its cgroup identity + // and its confirmation; record the newer plan (its generation) + // and the containers' state for status. + current.pod.Plan = p.Plan + current.pod.ContainersStarted = current.pod.ContainersStarted || p.ContainersStarted continue } // A pod binding arms no running task: a pod whose containers - // started relies on its container bindings, and a pod already bound - // keeps its last-good binding. - if p.ContainersStarted { - if current != nil { - current.pod.ContainersStarted = true - } + // started gets no new pod binding and relies on its container + // bindings. A bound pod whose plan changed moves to the new slot, + // so a restarted container is never judged by a superseded plan, + // but the move is not confirmed. + if p.ContainersStarted && current == nil { continue } slot, ok := b.slots[combination] @@ -445,25 +473,26 @@ func (b *PrivilegeBackend) BindPods(ctx context.Context, pods []sandboxpolicy.Po errs = append(errs, err) continue } - b.pods[uid] = &podBinding{pod: p, combination: combination, confirmed: true} + b.pods[uid] = &podBinding{pod: p, combination: combination, confirmed: !p.ContainersStarted} logger.L().Info("sandbox privilege pod bind", helpers.String("namespace", p.Namespace), helpers.String("pod", p.PodName)) } return errors.Join(append(errs, b.retire(ctx))...) } // BoundPodPlans returns the plans of the pod cgroups bound now (see -// sandboxpolicy.PodPlanReporter). An instance that is not ready protects -// nothing, so it reports none. +// sandboxpolicy.PodPlanReporter). While the instance is not ready, every pod +// plan is faulted: its protection cannot be confirmed. func (b *PrivilegeBackend) BoundPodPlans() (confirmed, faulted []sandboxpolicy.Plan) { b.mu.Lock() defer b.mu.Unlock() - if b.closed || b.instance == nil || b.instance.Ready() != nil { + if b.closed { return nil, nil } + ready := b.instance != nil && b.instance.Ready() == nil for _, p := range b.pods { slot, ok := b.slots[p.combination] switch { - case !ok || b.instance.SlotHealth(slot) != nil: + case !ready || !ok || b.instance.SlotHealth(slot) != nil: faulted = append(faulted, p.pod.Plan) case p.confirmed: confirmed = append(confirmed, p.pod.Plan) @@ -492,7 +521,11 @@ func (b *PrivilegeBackend) replaceInstance(ctx context.Context, prepared, extra var withdrawn []*privilegeBinding for k, v := range b.bindings { if v.pendingUnbind { - withdrawn = append(withdrawn, v) + // Under nextKey, next replaces the record, and Replace's commit + // closes the old lease when next does not reuse it. + if next == nil || k != nextKey { + withdrawn = append(withdrawn, v) + } continue } bindings[k] = v @@ -507,6 +540,9 @@ func (b *PrivilegeBackend) replaceInstance(ctx context.Context, prepared, extra for c, p := range extra { wanted[c] = p } + for c, p := range b.reserved { + wanted[c] = p + } for _, v := range bindings { if _, ok := wanted[v.combination]; !ok { wanted[v.combination] = b.combinations[v.combination] diff --git a/pkg/sandboxpolicy/igbackend/privilege_native.go b/pkg/sandboxpolicy/igbackend/privilege_native.go index cc00246be9..1d20bc5032 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_native.go +++ b/pkg/sandboxpolicy/igbackend/privilege_native.go @@ -60,8 +60,10 @@ func (i *nativePrivilegeInstance) Ready() error { } // SlotHealth fails on a fault counted in the slot, and, for a Strict slot, on -// a node fault: a child of an unconfirmed task can start in any cgroup -// (common/arming.h). An unreadable counter is a fault too. +// a node fault (ErrNodeFaultsOnly): a child of an unconfirmed task can start +// in any cgroup (common/arming.h). An unreadable counter is a fault too. An +// Audit slot ignores node faults by design: Audit coverage is reported, but a +// lost Audit judgement never blocks and never makes a policy NotReady. func (i *nativePrivilegeInstance) SlotHealth(slot uint32) error { n, err := i.bindings.SlotFaults(slot) if err != nil { @@ -78,7 +80,7 @@ func (i *nativePrivilegeInstance) SlotHealth(slot uint32) error { return err } if w.NodeFaults > 0 { - return fmt.Errorf("%d node tracking faults affect Strict privilege slot %d", w.NodeFaults, slot) + return fmt.Errorf("%w: %d affect Strict privilege slot %d", ErrNodeFaultsOnly, w.NodeFaults, slot) } return nil } diff --git a/pkg/sandboxpolicy/igbackend/privilege_test.go b/pkg/sandboxpolicy/igbackend/privilege_test.go index 5c3a2657b4..1edbc6f417 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_test.go +++ b/pkg/sandboxpolicy/igbackend/privilege_test.go @@ -607,3 +607,70 @@ func TestPrivilegePrepareReplaceMakesBindLive(t *testing.T) { t.Fatalf("second prepare: %v instances %d", err, len(e.instances)) } } + +// While the instance is not ready, every pod plan is faulted. +func TestPrivilegePodPlansFaultedWhileNotReady(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + p1 := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + if err := b.Prepare(ctx, []sandboxpolicy.Plan{p1}); err != nil { + t.Fatal(err) + } + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, p1)}); err != nil { + t.Fatal(err) + } + e.instances[0].readyErr = errors.New("not ready") + if plans, faulted := b.BoundPodPlans(); len(plans) != 0 || len(faulted) != 1 { + t.Fatalf("not ready: %d confirmed, %d faulted", len(plans), len(faulted)) + } +} + +// A bound pod whose containers started moves to its new slot when its plan +// changes, so a restarted container is never judged by a superseded plan; the +// move is not confirmed. +func TestPrivilegeStartedPodFollowsPlanChange(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + p1 := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + p2 := privPlan("r2", privPolicy("p1", api.ActionDeny)) + if err := b.Prepare(ctx, []sandboxpolicy.Plan{p1, p2}); err != nil { + t.Fatal(err) + } + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, p1)}); err != nil { + t.Fatal(err) + } + before := e.instances[0].bound[100] + moved := testPodBinding("a", 100, p2) + moved.ContainersStarted = true + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{moved}); err != nil { + t.Fatal(err) + } + if e.instances[0].bound[100] == before { + t.Fatal("the pod binding must move to the new plan's slot") + } + if plans, _ := b.BoundPodPlans(); len(plans) != 0 { + t.Fatal("a move after the containers started is not confirmed") + } +} + +// A Prepare between PrepareReplace and Replace keeps the reserved slot, so the +// Replace stays a live map write. +func TestPrivilegeReservedSlotSurvivesPrepare(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + p1 := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + p2 := privPlan("r2", privPolicy("p2", api.ActionDeny)) + if err := b.PrepareReplace(ctx, privBinding(1), p1); err != nil { + t.Fatal(err) + } + // The pod loop prepares another combination: a new instance. + if err := b.Prepare(ctx, []sandboxpolicy.Plan{p2}); err != nil || len(e.instances) != 2 { + t.Fatalf("prepare: %v instances %d", err, len(e.instances)) + } + if r, err := b.Replace(ctx, privBinding(1), p1); err != nil || !r.Committed || len(e.instances) != 2 { + t.Fatalf("replace must be live: %+v %v instances %d", r, err, len(e.instances)) + } +} diff --git a/pkg/sandboxpolicy/integration/bindings.go b/pkg/sandboxpolicy/integration/bindings.go index b50eb34572..2f92613056 100644 --- a/pkg/sandboxpolicy/integration/bindings.go +++ b/pkg/sandboxpolicy/integration/bindings.go @@ -67,17 +67,17 @@ func SnapshotSelectedBindingsFromContainers(ctx context.Context, client kubernet // node that are not finished, and the QoS class of each such pod (see // scheduledContainers). It feeds privilege slot // preparation and pod cgroup binding, not container binding. -func ScheduledContainers(ctx context.Context, client kubernetes.Interface, nodeName string) ([]sandboxpolicy.Binding, map[types.UID]corev1.PodQOSClass, error) { +func ScheduledContainers(ctx context.Context, client kubernetes.Interface, nodeName string) ([]sandboxpolicy.Binding, map[types.UID]corev1.PodQOSClass, map[types.UID]bool, error) { if client == nil || nodeName == "" { - return nil, nil, fmt.Errorf("Kubernetes client and node name are required") + return nil, nil, nil, fmt.Errorf("Kubernetes client and node name are required") } pods, err := client.CoreV1().Pods("").List(ctx, metav1.ListOptions{FieldSelector: fields.OneTermEqualSelector("spec.nodeName", nodeName).String()}) if err != nil { - return nil, nil, fmt.Errorf("list node pods: %w", err) + return nil, nil, nil, fmt.Errorf("list node pods: %w", err) } namespaces, err := client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) if err != nil { - return nil, nil, fmt.Errorf("list namespaces: %w", err) + return nil, nil, nil, fmt.Errorf("list namespaces: %w", err) } namespaceLabels := make(map[string]map[string]string, len(namespaces.Items)) for _, ns := range namespaces.Items { @@ -87,17 +87,18 @@ func ScheduledContainers(ctx context.Context, client kubernetes.Interface, nodeN for i := range pods.Items { list[i] = &pods.Items[i] } - result, qos := scheduledContainers(list, namespaceLabels, nodeName) - return result, qos, nil + result, qos, started := scheduledContainers(list, namespaceLabels, nodeName) + return result, qos, started, nil } // scheduledContainers is ScheduledContainers for a given pod list. A pod of a // namespace without labels is skipped. A mirror (static) pod has no QoS class // in the result: its pod cgroup is named after the kubelet's own UID, not the // API UID, so it gets no pod binding. -func scheduledContainers(pods []*corev1.Pod, namespaceLabels map[string]map[string]string, nodeName string) ([]sandboxpolicy.Binding, map[types.UID]corev1.PodQOSClass) { +func scheduledContainers(pods []*corev1.Pod, namespaceLabels map[string]map[string]string, nodeName string) ([]sandboxpolicy.Binding, map[types.UID]corev1.PodQOSClass, map[types.UID]bool) { var result []sandboxpolicy.Binding qos := map[types.UID]corev1.PodQOSClass{} + started := map[types.UID]bool{} for _, pod := range pods { if pod == nil || pod.Spec.NodeName != nodeName || pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { continue @@ -109,6 +110,9 @@ func scheduledContainers(pods []*corev1.Pod, namespaceLabels map[string]map[stri if _, mirror := pod.Annotations[corev1.MirrorPodAnnotationKey]; !mirror { qos[pod.UID] = pod.Status.QOSClass } + if podStarted(pod) { + started[pod.UID] = true + } for _, group := range []struct { kind string containers []corev1.Container @@ -121,7 +125,20 @@ func scheduledContainers(pods []*corev1.Pod, namespaceLabels map[string]map[stri result = append(result, sandboxpolicy.Binding{PodUID: pod.UID, Namespace: pod.Namespace, PodName: pod.Name, ContainerName: c.Name, Kind: "ephemeral", PodLabels: copyLabels(pod.Labels), NamespaceLabels: copyLabels(labels)}) } } - return result, qos + return result, qos, started +} + +// podStarted reports whether the API shows a container of the pod as created: +// a status with a container ID, a running or terminated state, or a restart. +func podStarted(pod *corev1.Pod) bool { + for _, statuses := range [][]corev1.ContainerStatus{pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses, pod.Status.EphemeralContainerStatuses} { + for _, st := range statuses { + if st.ContainerID != "" || st.State.Running != nil || st.State.Terminated != nil || st.RestartCount > 0 || (st.Started != nil && *st.Started) { + return true + } + } + } + return false } func snapshotBindings(ctx context.Context, client kubernetes.Interface, containers []*containercollection.Container, nodeName string, selector func(sandboxpolicy.Binding) bool) ([]sandboxpolicy.Binding, error) { diff --git a/pkg/sandboxpolicy/integration/bindings_test.go b/pkg/sandboxpolicy/integration/bindings_test.go index 203472deeb..00279e1b43 100644 --- a/pkg/sandboxpolicy/integration/bindings_test.go +++ b/pkg/sandboxpolicy/integration/bindings_test.go @@ -257,3 +257,39 @@ func TestSnapshotRejectsInvalidMembershipDespiteMountedPath(t *testing.T) { }) } } + +func TestPodStarted(t *testing.T) { + started := true + for name, tc := range map[string]struct { + status corev1.PodStatus + want bool + }{ + "no status": {corev1.PodStatus{}, false}, + "waiting": {corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{}}}}}, false}, + "container ID": {corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{ContainerID: "containerd://abc"}}}, true}, + "running": {corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}}}, true}, + "init done": {corev1.PodStatus{InitContainerStatuses: []corev1.ContainerStatus{{State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{}}}}}, true}, + "started": {corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Started: &started}}}, true}, + } { + if got := podStarted(&corev1.Pod{Status: tc.status}); got != tc.want { + t.Errorf("%s: %v, want %v", name, got, tc.want) + } + } +} + +func TestUnderPod(t *testing.T) { + pod := "/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podabc.slice" + containers := []string{"/system.slice/docker.service", pod} + if underPod(containers, pod) { + t.Fatal("the pod cgroup itself is not a container under it") + } + if !underPod(append(containers, pod+"/cri-containerd-1.scope"), pod) { + t.Fatal("a container cgroup under the pod cgroup") + } + if !underPod([]string{"/host" + pod + "/cri-containerd-1.scope"}, pod) { + t.Fatal("a longer prefix still matches") + } + if underPod([]string{pod + "x/cri-containerd-1.scope"}, pod) { + t.Fatal("a sibling with the same prefix does not match") + } +} diff --git a/pkg/sandboxpolicy/integration/service.go b/pkg/sandboxpolicy/integration/service.go index 46fddf79a5..e8284674b6 100644 --- a/pkg/sandboxpolicy/integration/service.go +++ b/pkg/sandboxpolicy/integration/service.go @@ -6,6 +6,7 @@ import ( "fmt" stdmaps "maps" "path/filepath" + "strings" "sync" "sync/atomic" "time" @@ -54,7 +55,7 @@ type ServiceOptions struct { // PrivilegeImageName enables the privilege section. Empty rejects it. PrivilegeImageName string // QualifiedKernels lists the kernel releases (uname -r) on which the - // privilege section may report Ready; nil uses igbackend.QualifiedKernels, + // privilege section may report Ready; empty uses igbackend.QualifiedKernels, // and "*" admits every kernel. On another kernel it enforces but stays // NotReady (PrivilegeKernelUnqualified). QualifiedKernels []string @@ -93,6 +94,7 @@ type Service struct { // from the last slot preparation, used when there are no informers. scheduled []sandboxpolicy.Binding qos map[types.UID]corev1.PodQOSClass + started map[types.UID]bool // podFirstSeen is when a pod with a pod plan was first seen. podPolling is // set while such a pod has no pod cgroup yet, for up to prepareInterval. podFirstSeen map[types.UID]time.Time @@ -101,6 +103,7 @@ type Service struct { // reused by the poll. podPollErr is the last error the poll reported. podPods []sandboxpolicy.Binding podQoS map[types.UID]corev1.PodQOSClass + podStarted map[types.UID]bool podPlans map[types.UID]sandboxpolicy.Plan podPollErr string // podCgroupID returns the cgroup ID of a pod cgroup path (replaced in tests). @@ -206,7 +209,7 @@ func NewService(ctx context.Context, options ServiceOptions) (*Service, error) { if privilege != nil { release, rerr := igbackend.KernelRelease() if rerr != nil { - sections.SetPrivilegeUnqualified(&sandboxpolicy.EnforcementUnavailableError{Section: "privilege", Reason: "PrivilegeKernelUnqualified", Detail: fmt.Sprintf("kernel release: %v", rerr)}) + sections.SetPrivilegeUnqualified(&sandboxpolicy.EnforcementUnavailableError{Section: "privilege", Reason: "PrivilegeKernelUnqualified", Enforcing: true, Detail: fmt.Sprintf("kernel release: %v", rerr)}) } else { sections.SetPrivilegeUnqualified(igbackend.CheckQualifiedKernel(release, options.QualifiedKernels)) } @@ -367,9 +370,9 @@ func (s *Service) refresh(ctx context.Context) { if s.options.PrivilegeImageName != "" && s.options.Kubernetes != nil && s.Controller.HasPrivilegePolicies() && (s.podsScheduled.Swap(false) || time.Since(s.lastPrepare) >= prepareInterval) { s.lastPrepare = time.Now() - scheduled, qos, err := s.scheduledPods(ctx) + scheduled, qos, started, err := s.scheduledPods(ctx) if err == nil { - s.scheduled, s.qos = scheduled, qos + s.scheduled, s.qos, s.started = scheduled, qos, started err = s.Controller.PrepareScheduled(ctx, scheduled) } if err != nil { @@ -421,10 +424,10 @@ func (s *Service) bindPods(ctx context.Context) { return } s.podPolling = false // set again below when a pod cgroup is missing - scheduled, qos := s.scheduled, s.qos + scheduled, qos, started := s.scheduled, s.qos, s.started if s.pods != nil { var err error - if scheduled, qos, err = s.scheduledPods(ctx); err != nil { + if scheduled, qos, started, err = s.scheduledPods(ctx); err != nil { s.report(fmt.Errorf("privilege pod binding: %w", err)) return } @@ -434,9 +437,9 @@ func (s *Service) bindPods(ctx context.Context) { s.podPlans = nil return // the policies are not known: keep the pod bindings } - s.podPods, s.podQoS, s.podPlans = scheduled, qos, plans + s.podPods, s.podQoS, s.podStarted, s.podPlans = scheduled, qos, started, plans s.podPollErr = "" - s.bindPodPlans(ctx, scheduled, qos, plans, false) + s.bindPodPlans(ctx, scheduled, qos, started, plans, false) } // pollPods checks again for the pod cgroups that were missing. It reuses the @@ -447,10 +450,10 @@ func (s *Service) pollPods(ctx context.Context) { if s.podPlans == nil { return } - s.bindPodPlans(ctx, s.podPods, s.podQoS, s.podPlans, true) + s.bindPodPlans(ctx, s.podPods, s.podQoS, s.podStarted, s.podPlans, true) } -func (s *Service) bindPodPlans(ctx context.Context, scheduled []sandboxpolicy.Binding, qos map[types.UID]corev1.PodQOSClass, known map[types.UID]sandboxpolicy.Plan, poll bool) { +func (s *Service) bindPodPlans(ctx context.Context, scheduled []sandboxpolicy.Binding, qos map[types.UID]corev1.PodQOSClass, apiStarted map[types.UID]bool, known map[types.UID]sandboxpolicy.Plan, poll bool) { binder := s.backend.(sandboxpolicy.PodBinder) plans := make(map[types.UID]sandboxpolicy.Plan, len(known)) for k, v := range known { @@ -461,7 +464,7 @@ func (s *Service) bindPodPlans(ctx context.Context, scheduled []sandboxpolicy.Bi firstSeen := map[types.UID]time.Time{} polling := false if len(plans) > 0 { - layout, known, containerCgroups, started := s.podCgroupLayout() + layout, known, containerCgroups, started, containerPaths := s.podCgroupLayout() for _, c := range scheduled { plan, ok := plans[c.PodUID] if !ok { @@ -487,7 +490,8 @@ func (s *Service) bindPodPlans(ctx context.Context, scheduled []sandboxpolicy.Bi if containerCgroups[id] { continue } - pods = append(pods, sandboxpolicy.PodBinding{PodUID: c.PodUID, Namespace: c.Namespace, PodName: c.PodName, CgroupID: id, CgroupPath: path, Plan: plan, ContainersStarted: started[c.PodUID]}) + pods = append(pods, sandboxpolicy.PodBinding{PodUID: c.PodUID, Namespace: c.Namespace, PodName: c.PodName, CgroupID: id, CgroupPath: path, Plan: plan, + ContainersStarted: started[c.PodUID] || apiStarted[c.PodUID] || underPod(containerPaths, path)}) } } s.podFirstSeen, s.podPolling = firstSeen, polling @@ -504,39 +508,41 @@ func (s *Service) bindPodPlans(ctx context.Context, scheduled []sandboxpolicy.Bi // scheduledPods returns the containers and QoS classes of the node's pods (see // ScheduledContainers), from the informer caches when there are informers. -func (s *Service) scheduledPods(ctx context.Context) ([]sandboxpolicy.Binding, map[types.UID]corev1.PodQOSClass, error) { +func (s *Service) scheduledPods(ctx context.Context) ([]sandboxpolicy.Binding, map[types.UID]corev1.PodQOSClass, map[types.UID]bool, error) { if s.pods == nil { - scheduled, qos, err := ScheduledContainers(ctx, s.options.Kubernetes, s.options.NodeName) - return scheduled, qos, err + return ScheduledContainers(ctx, s.options.Kubernetes, s.options.NodeName) } pods, err := s.pods.Core().V1().Pods().Lister().List(labels.Everything()) if err != nil { - return nil, nil, err + return nil, nil, nil, err } namespaces, err := s.namespaces.Core().V1().Namespaces().Lister().List(labels.Everything()) if err != nil { - return nil, nil, err + return nil, nil, nil, err } namespaceLabels := make(map[string]map[string]string, len(namespaces)) for _, ns := range namespaces { namespaceLabels[ns.Name] = ns.Labels } - scheduled, qos := scheduledContainers(pods, namespaceLabels, s.options.NodeName) - return scheduled, qos, nil + scheduled, qos, started := scheduledContainers(pods, namespaceLabels, s.options.NodeName) + return scheduled, qos, started, nil } // podCgroupLayout reads the pod cgroup layout from a running pod container, // and returns the cgroup IDs of the running containers. -func (s *Service) podCgroupLayout() (podCgroupLayout, bool, map[uint64]bool, map[types.UID]bool) { +func (s *Service) podCgroupLayout() (podCgroupLayout, bool, map[uint64]bool, map[types.UID]bool, []string) { var layout podCgroupLayout found := false cgroups := map[uint64]bool{} - // The pods with a container that exists: the collection learns a - // container when the runtime creates it, before its workload runs, and - // it drops pause containers. + // The pods with a container that exists, by pod UID and by cgroup path + // (a container whose pod UID is not enriched yet). The collection drops + // pause containers. Together with the pod's API status this is a + // snapshot: a container created after it is not seen until the next + // call (see sandboxpolicy.PodBinding). started := map[types.UID]bool{} + var paths []string if s.options.Containers == nil { - return layout, false, cgroups, started + return layout, false, cgroups, started, paths } s.options.Containers.ContainerRange(func(c *containercollection.Container) { if c == nil { @@ -546,11 +552,30 @@ func (s *Service) podCgroupLayout() (podCgroupLayout, bool, map[uint64]bool, map if c.K8s.PodUID != "" { started[types.UID(c.K8s.PodUID)] = true } + if c.CgroupV2 != "" { + paths = append(paths, c.CgroupV2) + } if !found { layout, found = learnPodCgroupLayout(c.CgroupV2, types.UID(c.K8s.PodUID)) } }) - return layout, found, cgroups, started + return layout, found, cgroups, started, paths +} + +// underPod reports whether a container cgroup path lies under the pod cgroup +// path. Both come from the same host view; the container path may carry a +// longer prefix, so the pod path is matched as a path segment sequence. +func underPod(containerPaths []string, podPath string) bool { + pod := "/" + strings.Trim(podPath, "/") + "/" + if pod == "//" { + return false + } + for _, p := range containerPaths { + if strings.Contains(p+"/", pod) && !strings.HasSuffix(strings.TrimRight(p, "/")+"/", pod) { + return true + } + } + return false } // ContainerCallback is deliberately nonblocking: event delivery must never hold diff --git a/pkg/sandboxpolicy/sections.go b/pkg/sandboxpolicy/sections.go index 7f59d03e2f..6f60022b4e 100644 --- a/pkg/sandboxpolicy/sections.go +++ b/pkg/sandboxpolicy/sections.go @@ -241,14 +241,33 @@ func (s *SectionBackend) Replace(ctx context.Context, binding Binding, plan Plan return ReplaceResult{}, fmt.Errorf("privilege section: %w", err) } } - e := apply(ctx, s.exec, binding, old.exec, want.exec) - if !e.committed { - return ReplaceResult{ProtectionUnconfirmed: e.unconfirmed}, fmt.Errorf("exec section: %w", e.err) - } oldPrivilege := old.privilege if old.privilegeCleanup && oldPrivilege.Revision == "" { oldPrivilege.Revision = "cleanup" // forces a privilege Withdraw retry } + if want.privilege.Revision == "" { + // The privilege section is removed: withdraw it first. A withdraw + // that does not commit (the instance is not ready) changes nothing + // and is retried, with no exec commit and rollback each time. + p := apply(ctx, s.privilege, binding, oldPrivilege, want.privilege) + if !p.committed { + return ReplaceResult{ProtectionUnconfirmed: p.unconfirmed}, fmt.Errorf("privilege section: %w", p.err) + } + e := apply(ctx, s.exec, binding, old.exec, want.exec) + if !e.committed { + // Exec keeps its last-good plan; the privilege removal stays. + old.privilege, old.privilegeCleanup = Plan{}, p.err != nil + s.active[k] = old + return ReplaceResult{ProtectionUnconfirmed: e.unconfirmed || p.unconfirmed}, fmt.Errorf("exec section: %w", e.err) + } + want.privilegeCleanup = p.err != nil + s.active[k] = want + return ReplaceResult{Committed: true, ActiveRevision: plan.Revision, ProtectionUnconfirmed: e.unconfirmed || p.unconfirmed}, errors.Join(e.err, p.err) + } + e := apply(ctx, s.exec, binding, old.exec, want.exec) + if !e.committed { + return ReplaceResult{ProtectionUnconfirmed: e.unconfirmed}, fmt.Errorf("exec section: %w", e.err) + } p := apply(ctx, s.privilege, binding, oldPrivilege, want.privilege) if !p.committed { // The live bind failed after a successful prepare (identity check @@ -257,7 +276,6 @@ func (s *SectionBackend) Replace(ctx context.Context, binding Binding, plan Plan return ReplaceResult{ProtectionUnconfirmed: e.unconfirmed || p.unconfirmed || back.unconfirmed || !back.committed}, errors.Join(fmt.Errorf("privilege section: %w", p.err), back.err) } - want.privilegeCleanup = want.privilege.Revision == "" && p.err != nil s.active[k] = want return ReplaceResult{Committed: true, ActiveRevision: plan.Revision, ProtectionUnconfirmed: e.unconfirmed || p.unconfirmed}, errors.Join(e.err, p.err) } diff --git a/pkg/sandboxpolicy/sections_test.go b/pkg/sandboxpolicy/sections_test.go index d60f6cdf79..1159f4f36d 100644 --- a/pkg/sandboxpolicy/sections_test.go +++ b/pkg/sandboxpolicy/sections_test.go @@ -451,3 +451,48 @@ func TestSectionBackendPreparesPrivilegeBeforeExec(t *testing.T) { t.Fatalf("prepares %v", privilege.prepared) } } + +type pendingWithdrawBackend struct { + fakeBackend + pending bool +} + +func (b *pendingWithdrawBackend) Withdraw(ctx context.Context, binding Binding) (WithdrawResult, error) { + if b.pending { + return WithdrawResult{ProtectionUnconfirmed: true}, errors.New("unbind pending") + } + return b.fakeBackend.Withdraw(ctx, binding) +} + +// Removing the privilege section withdraws it before exec commits: a pending +// withdraw changes no exec plan, so exec is not reloaded and rolled back on +// each retry. +func TestSectionBackendPendingPrivilegeRemovalKeepsExec(t *testing.T) { + exec := &fakeBackend{plans: map[string]Plan{}} + privilege := &pendingWithdrawBackend{fakeBackend: fakeBackend{plans: map[string]Plan{}}} + s, _ := NewSectionBackend(exec, privilege, nil) + b := Binding{ContainerID: "c"} + execPolicy := CompiledPolicy{Revision: "e", Exec: api.ExecSection{Default: api.ActionDeny}} + privPolicy := CompiledPolicy{Revision: "p", Privilege: privilegeSection(api.ActionAllow)} + if _, err := s.Replace(context.Background(), b, sectionPlan("r1", execPolicy, privPolicy)); err != nil { + t.Fatal(err) + } + privilege.pending = true + execPolicy.Revision = "e2" + execPolicy.Exec.Default = api.ActionAllow + for i := 0; i < 2; i++ { + if r, err := s.Replace(context.Background(), b, sectionPlan("r2", execPolicy)); err == nil || r.Committed { + t.Fatalf("pending removal: %+v %v", r, err) + } + } + if exec.replacements != 1 || exec.withdrawals != 0 { + t.Fatalf("exec must not change while the removal is pending: %d replacements, %d withdrawals", exec.replacements, exec.withdrawals) + } + privilege.pending = false + if r, err := s.Replace(context.Background(), b, sectionPlan("r2", execPolicy)); err != nil || !r.Committed { + t.Fatalf("retry: %+v %v", r, err) + } + if exec.replacements != 2 { + t.Fatalf("exec commits after the removal: %d", exec.replacements) + } +} From 033fa79de71a74059fd8d65ca84b6e9bd335709a Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:09:43 +0200 Subject: [PATCH 53/54] fix(sandboxpolicy): reserve only a prepared slot that exists; recover a pending withdraw - PrepareReplace reserves its combination only after the slot exists, so a combination that does not fit never blocks other workloads. - A withdraw on an instance that is not ready replaces it after healRetry, so a pending withdraw ends even when no privilege policy is left. - Exec failing after the privilege removal reports protection unconfirmed. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- pkg/sandboxpolicy/igbackend/privilege.go | 21 +++++++++- pkg/sandboxpolicy/igbackend/privilege_test.go | 40 +++++++++++++++++++ pkg/sandboxpolicy/sections.go | 4 +- 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/pkg/sandboxpolicy/igbackend/privilege.go b/pkg/sandboxpolicy/igbackend/privilege.go index 1f33d45ba4..9bf0e64ca5 100644 --- a/pkg/sandboxpolicy/igbackend/privilege.go +++ b/pkg/sandboxpolicy/igbackend/privilege.go @@ -308,8 +308,11 @@ func (b *PrivilegeBackend) PrepareReplace(ctx context.Context, _ sandboxpolicy.B if len(policies) == 0 { return fmt.Errorf("plan has no privilege section") } - b.reserved = map[string][]sandboxpolicy.CompiledPolicy{combination: policies} + // Reserved only when the slot exists: a combination that does not fit + // must never be carried into other workloads' replacements. + b.reserved = nil if _, ok := b.slots[combination]; ok && b.instance != nil && b.instance.Ready() == nil { + b.reserved = map[string][]sandboxpolicy.CompiledPolicy{combination: policies} return nil } extra := map[string][]sandboxpolicy.CompiledPolicy{combination: policies} @@ -317,6 +320,9 @@ func (b *PrivilegeBackend) PrepareReplace(ctx context.Context, _ sandboxpolicy.B if err != nil && len(b.prepared) > 0 { err = b.replaceInstance(ctx, nil, extra, identity{}, nil, true) } + if err == nil { + b.reserved = extra + } return errors.Join(err, b.retire(ctx)) } @@ -333,7 +339,18 @@ func (b *PrivilegeBackend) Withdraw(ctx context.Context, binding sandboxpolicy.B if b.instance != nil { if err := b.instance.Ready(); err != nil { current.pendingUnbind = true - return sandboxpolicy.WithdrawResult{ProtectionUnconfirmed: true}, fmt.Errorf("privilege unbind pending: %w", err) + // A new instance never binds a pending record; replace the + // instance that is not ready, at most every healRetry, so a + // pending withdraw cannot wait forever. + if time.Since(b.lastHeal) < healRetry { + return sandboxpolicy.WithdrawResult{ProtectionUnconfirmed: true}, fmt.Errorf("privilege unbind pending: %w", err) + } + b.lastHeal = time.Now() + if rerr := b.replaceInstance(ctx, b.prepared, nil, identity{}, nil, true); rerr != nil { + return sandboxpolicy.WithdrawResult{ProtectionUnconfirmed: true}, errors.Join(fmt.Errorf("privilege unbind pending: %w", err), rerr) + } + // replaceInstance dropped the record and closed its lease. + return sandboxpolicy.WithdrawResult{Committed: true}, b.retire(ctx) } if err := b.instance.Unbind(native(binding)); err != nil { current.pendingUnbind = true diff --git a/pkg/sandboxpolicy/igbackend/privilege_test.go b/pkg/sandboxpolicy/igbackend/privilege_test.go index 1edbc6f417..df15e02219 100644 --- a/pkg/sandboxpolicy/igbackend/privilege_test.go +++ b/pkg/sandboxpolicy/igbackend/privilege_test.go @@ -6,6 +6,7 @@ import ( "fmt" "strings" "testing" + "time" "github.com/kubescape/node-agent/pkg/ebpf/gadgets/common/enforcement" privruntime "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/runtime" @@ -203,6 +204,7 @@ func TestPrivilegeWithdrawPendingWhenInstanceNotReady(t *testing.T) { t.Fatal(err) } e.instances[0].readyErr = errors.New("not ready") + b.lastHeal = time.Now() // no recovery replacement in this part of the test if r, err := b.Withdraw(ctx, privBinding(1)); err == nil || r.Committed || !r.ProtectionUnconfirmed { t.Fatalf("withdraw on an instance that is not ready must stay pending: %+v %v", r, err) } @@ -220,6 +222,7 @@ func TestPrivilegeWithdrawPendingWhenInstanceNotReady(t *testing.T) { t.Fatal(err) } e.instances[0].readyErr = errors.New("not ready") + b.lastHeal = time.Now() if r, _ := b.Withdraw(ctx, privBinding(1)); r.Committed { t.Fatal("must stay pending") } @@ -674,3 +677,40 @@ func TestPrivilegeReservedSlotSurvivesPrepare(t *testing.T) { t.Fatalf("replace must be live: %+v %v instances %d", r, err, len(e.instances)) } } + +// After healRetry, a withdraw on an instance that is not ready replaces it; +// the new instance never binds the withdrawn container. +func TestPrivilegePendingWithdrawReplacesInstance(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + if _, err := b.Replace(ctx, privBinding(1), privPlan("r1", privPolicy("p1", api.ActionAllow))); err != nil { + t.Fatal(err) + } + e.instances[0].readyErr = errors.New("not ready") + if r, err := b.Withdraw(ctx, privBinding(1)); err != nil || !r.Committed || len(e.instances) != 2 { + t.Fatalf("recovery: %+v %v instances %d", r, err, len(e.instances)) + } + if _, bound := e.instances[1].bound[1]; bound { + t.Fatal("the new instance must not bind the withdrawn container") + } +} + +// A PrepareReplace whose combination does not fit reserves nothing, so it +// never blocks another workload's replacement. +func TestPrivilegeFailedPrepareReservesNothing(t *testing.T) { + big := privPlan("big", privPolicy("big", api.ActionDeny)) + e := &fakePrivilegeEngine{failIf: func(c privruntime.Config) bool { return c.Policy.Slots > 1 }} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + p1 := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + if _, err := b.Replace(ctx, privBinding(1), p1); err != nil { + t.Fatal(err) + } + if err := b.PrepareReplace(ctx, privBinding(2), big); err == nil { + t.Fatal("a combination that does not fit must fail") + } + if b.reserved != nil { + t.Fatalf("nothing may be reserved: %v", b.reserved) + } +} diff --git a/pkg/sandboxpolicy/sections.go b/pkg/sandboxpolicy/sections.go index 6f60022b4e..a1542ec0cf 100644 --- a/pkg/sandboxpolicy/sections.go +++ b/pkg/sandboxpolicy/sections.go @@ -256,9 +256,11 @@ func (s *SectionBackend) Replace(ctx context.Context, binding Binding, plan Plan e := apply(ctx, s.exec, binding, old.exec, want.exec) if !e.committed { // Exec keeps its last-good plan; the privilege removal stays. + // The binding now runs neither the last-good nor the desired + // plan, so its protection is unconfirmed. old.privilege, old.privilegeCleanup = Plan{}, p.err != nil s.active[k] = old - return ReplaceResult{ProtectionUnconfirmed: e.unconfirmed || p.unconfirmed}, fmt.Errorf("exec section: %w", e.err) + return ReplaceResult{ProtectionUnconfirmed: true}, fmt.Errorf("exec section: %w", e.err) } want.privilegeCleanup = p.err != nil s.active[k] = want From 1284c1bcf308f1d29ea8ba7dc3f06cab9180a368 Mon Sep 17 00:00:00 2001 From: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:21:41 +0200 Subject: [PATCH 54/54] fix(sandboxpolicy): a partial privilege removal never attests its old revision When exec fails after the privilege removal committed, the binding runs neither plan. Its record now has no revision, so Attest fails until a Replace commits. The built-in qualified kernel list holds only the kernel where the kernel tests passed (6.12.113-138.209); 6.18.56 joins after its run. Signed-off-by: Alon Liwsky <40373481+AlonLiwsky@users.noreply.github.com> --- docs/features/sandbox-policy.md | 5 +++-- pkg/sandboxpolicy/igbackend/kernel.go | 6 +++--- pkg/sandboxpolicy/sections.go | 3 ++- pkg/sandboxpolicy/status_test.go | 23 +++++++++++++++++++++++ 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/docs/features/sandbox-policy.md b/docs/features/sandbox-policy.md index ddcd686af8..a62ca948c9 100644 --- a/docs/features/sandbox-policy.md +++ b/docs/features/sandbox-policy.md @@ -96,8 +96,9 @@ minutes, because their cause may persist. The privilege section reports Ready only on a qualified kernel: `sandboxPolicyQualifiedKernels` lists kernel releases (`uname -r`); unset or empty uses -the AL2023 amd64 fleet kernels `6.12.113-138.209.amzn2023.x86_64` and -`6.18.56-125.164.amzn2023.x86_64`, and `["*"]` admits every kernel. On another +the built-in list of kernels where the kernel tests passed, today the AL2023 amd64 +fleet kernel `6.12.113-138.209.amzn2023.x86_64` (`6.18.56-125.164.amzn2023.x86_64` +joins after its run), and `["*"]` admits every kernel. On another kernel the section still enforces, but every privilege policy reports `Ready=False` with reason `PrivilegeKernelUnqualified`. diff --git a/pkg/sandboxpolicy/igbackend/kernel.go b/pkg/sandboxpolicy/igbackend/kernel.go index 944f97f4b9..3ef505f577 100644 --- a/pkg/sandboxpolicy/igbackend/kernel.go +++ b/pkg/sandboxpolicy/igbackend/kernel.go @@ -9,11 +9,11 @@ import ( "golang.org/x/sys/unix" ) -// QualifiedKernels are the kernel releases (uname -r) on which the privilege -// section passed its kernel qualification: the AL2023 amd64 fleet kernels. +// QualifiedKernels are the kernel releases (uname -r) on which the section +// kernel tests passed (AL2023 amd64 fleet kernels). 6.18.56-125.164 joins +// after its run. var QualifiedKernels = []string{ "6.12.113-138.209.amzn2023.x86_64", - "6.18.56-125.164.amzn2023.x86_64", } // AnyKernel in a qualified list admits every kernel (test clusters). diff --git a/pkg/sandboxpolicy/sections.go b/pkg/sandboxpolicy/sections.go index a1542ec0cf..41dd29e77c 100644 --- a/pkg/sandboxpolicy/sections.go +++ b/pkg/sandboxpolicy/sections.go @@ -258,7 +258,8 @@ func (s *SectionBackend) Replace(ctx context.Context, binding Binding, plan Plan // Exec keeps its last-good plan; the privilege removal stays. // The binding now runs neither the last-good nor the desired // plan, so its protection is unconfirmed. - old.privilege, old.privilegeCleanup = Plan{}, p.err != nil + // No revision: Attest fails until a Replace commits. + old.privilege, old.privilegeCleanup, old.revision = Plan{}, p.err != nil, "" s.active[k] = old return ReplaceResult{ProtectionUnconfirmed: true}, fmt.Errorf("exec section: %w", e.err) } diff --git a/pkg/sandboxpolicy/status_test.go b/pkg/sandboxpolicy/status_test.go index fca791d66d..7d03c98510 100644 --- a/pkg/sandboxpolicy/status_test.go +++ b/pkg/sandboxpolicy/status_test.go @@ -212,3 +212,26 @@ func TestUnqualifiedKernelIsNotReady(t *testing.T) { t.Fatalf("the privilege section still enforces: %v", err) } } + +// Exec failing after the privilege removal leaves a binding that runs neither +// plan: its old revision must not attest. +func TestSectionBackendPartialRemovalDoesNotAttest(t *testing.T) { + exec := &fakeBackend{plans: map[string]Plan{}} + privilege := &fakeBackend{plans: map[string]Plan{}} + s, _ := NewSectionBackend(exec, privilege, nil) + b := Binding{ContainerID: "c"} + execPolicy := CompiledPolicy{Revision: "e", Exec: api.ExecSection{Default: api.ActionDeny}} + privPolicy := CompiledPolicy{Revision: "p", Privilege: privilegeSection(api.ActionAllow)} + if _, err := s.Replace(context.Background(), b, sectionPlan("r1", execPolicy, privPolicy)); err != nil { + t.Fatal(err) + } + exec.fail = true + execPolicy.Revision = "e2" + execPolicy.Exec.Default = api.ActionAllow + if r, err := s.Replace(context.Background(), b, sectionPlan("r2", execPolicy)); err == nil || r.Committed || !r.ProtectionUnconfirmed { + t.Fatalf("exec failure after the removal: %+v %v", r, err) + } + if err := s.Attest(context.Background(), b, "r1"); err == nil { + t.Fatal("the old revision must not attest after a partial removal") + } +}