diff --git a/cmd/main.go b/cmd/main.go index 3886794e10..5c56c22c19 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -577,6 +577,7 @@ func main() { Kubernetes: k8sClient.GetKubernetesClient(), Dynamic: k8sClient.GetDynamicClient(), Containers: mainHandler.GetContainerCollection(), Tracers: mainHandler.GetTracerCollection(), NodeName: cfg.NodeName, AgentNamespace: cfg.NamespaceName, ImageName: cfg.SandboxPolicyImage, ImageArchive: "tracers.tar", + PrivilegeImageName: cfg.SandboxPolicyPrivilegeImage, QualifiedKernels: cfg.SandboxPolicyQualifiedKernels, ContainerdSocket: cfg.SandboxPolicyContainerdSocket, ControlRoot: cfg.SandboxPolicyControlRoot, MaxOwners: cfg.SandboxPolicyMaxOwners, Sources: sandboxintegration.NewProfileResolver(storageClient.GetStorageClient()), Receivers: thirdPartyTracers.ThirdPartyEventReceivers, diff --git a/docs/features/sandbox-policy.md b/docs/features/sandbox-policy.md index 95200b0f3f..a62ca948c9 100644 --- a/docs/features/sandbox-policy.md +++ b/docs/features/sandbox-policy.md @@ -68,6 +68,47 @@ This change does not grant chart RBAC, change a chart version or store policy in SQLite/files. Kubernetes owns the CRD objects. Per-policy/node status resources are output; the agent does not write parent aggregate status. +Each report carries `appliedGeneration`, the oldest policy generation still +installed on the node, counting container bindings and privilege pod cgroup +bindings. It stays on the last-good generation while a newer revision is +rejected, and is absent when no binding enforces the policy. `stale` is set only +when an installed binding lags the desired generation or its protection cannot be +confirmed: a failure that leaves the current generation enforced is `Ready=False` +with `stale: false`. At +startup the agent reads the host's active LSM list once. Every section enforces +through BPF LSM, so when `bpf` is missing every report says `tier: none`, and +every enabled policy reports `Ready=False` with reason `BPFLSMInactive`. + +A privilege pod cgroup binding counts only where the witness counters confirm +it: the pod was bound before any of its containers existed, in the instance +that enforces now, and its slot has no tracking fault. "Its containers exist" +comes from the pod's API status and from the container collection (by pod UID +and by cgroup path); a container created after that snapshot is seen at the +next call. A pod whose containers already exist gets no pod binding; its +containers rely on their own bindings. A bound pod whose plan changes later +moves to its new slot, unconfirmed. While the instance is not ready, every pod +binding is unconfirmed. +A pod binding whose slot has a fault makes the policy report +`ProtectionUnconfirmed`. For a Strict slot a node tracking fault (a lost child +mark of a `CLONE_INTO_CGROUP` fork anywhere on the node) is a fault too; a new +instance clears it; node faults alone start a new instance at most every 10 +minutes, because their cause may persist. + +The privilege section reports Ready only on a qualified kernel: +`sandboxPolicyQualifiedKernels` lists kernel releases (`uname -r`); unset or empty uses +the built-in list of kernels where the kernel tests passed, today the AL2023 amd64 +fleet kernel `6.12.113-138.209.amzn2023.x86_64` (`6.18.56-125.164.amzn2023.x86_64` +joins after its run), and `["*"]` admits every kernel. On another +kernel the section still enforces, but every privilege policy reports +`Ready=False` with reason `PrivilegeKernelUnqualified`. + +A privilege change commits in two phases: the privilege slot is prepared first +(this can replace the node instance and binds nothing), then exec commits, then +the privilege bind is a live map write. A failed prepare leaves both sections on +their last-good plan. Removing the privilege section withdraws it first: a +withdraw on an instance that is not ready stays pending and is retried, and exec +does not change meanwhile. + ## Reconciliation and failure behavior Configuration watch events are **refresh hints**, not trusted queued payloads. diff --git a/go.mod b/go.mod index ddd2494859..d06087b9d4 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/kubescape/backend v0.0.39 github.com/kubescape/go-logger v0.0.35 github.com/kubescape/k8s-interface v0.0.214 - github.com/kubescape/storage v0.0.388-0.20261105083812-136b4c98606f + github.com/kubescape/storage v0.0.388-0.20261109155453-fa55b25a70a3 github.com/kubescape/workerpool v0.0.0-20250526074519-0e4a4e7f44cf github.com/moby/sys/mountinfo v0.7.2 github.com/opcoder0/fanotify v0.4.2 diff --git a/go.sum b/go.sum index e8aa7e6027..fb5471c377 100644 --- a/go.sum +++ b/go.sum @@ -1459,6 +1459,8 @@ github.com/kubescape/storage v0.0.352-0.20261013133511-eb6406e01cb9 h1:pPFehIHOZ github.com/kubescape/storage v0.0.352-0.20261013133511-eb6406e01cb9/go.mod h1:OSYXj9TzVAuuz+Gn2bIzp3Vg8rofTPH7x5Asek0N7mE= github.com/kubescape/storage v0.0.388-0.20261105083812-136b4c98606f h1:KTA6ctZBLIi+XRZwKFtl8etQZOTPciSy/nfAtBizzlc= github.com/kubescape/storage v0.0.388-0.20261105083812-136b4c98606f/go.mod h1:z3uiLdH1O3+eNiKuEOOalJy9uqH7ggNepaUSSWWm5ps= +github.com/kubescape/storage v0.0.388-0.20261109155453-fa55b25a70a3 h1:oSyi5Tiixb+25/HMG03DiKtmiQiyQbuXs0p1F/WKfik= +github.com/kubescape/storage v0.0.388-0.20261109155453-fa55b25a70a3/go.mod h1:z3uiLdH1O3+eNiKuEOOalJy9uqH7ggNepaUSSWWm5ps= github.com/kubescape/syft v1.32.0-ks.2 h1:xdUksUmKEyyVKsTfJDYW8Z5HawVJtelsUolPOsWtDx0= github.com/kubescape/syft v1.32.0-ks.2/go.mod h1:E6Kd4iBM2ljUOUQvSt7hVK6vBwaHkMXwcvBZmGMSY5o= github.com/kubescape/workerpool v0.0.0-20250526074519-0e4a4e7f44cf h1:hI0jVwrB6fT4GJWvuUjzObfci1CUknrZdRHfnRVtKM0= @@ -1863,6 +1865,7 @@ github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8r github.com/sebdah/goldie/v2 v2.8.0/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= github.com/seccomp/libseccomp-golang v0.11.0 h1:SDkcBRqGLP+sezmMACkxO1EfgbghxIxnRKfd6mHUEis= github.com/seccomp/libseccomp-golang v0.11.0/go.mod h1:5m1Lk8E9OwgZTTVz4bBOer7JuazaBa+xTkM895tDiWc= +github.com/seccomp/libseccomp-golang v0.11.1 h1:wuk4ZjSx6kyQII4rj6G6fvVzRHQaSiPvccJazDagu4g= github.com/seccomp/libseccomp-golang v0.11.1/go.mod h1:5m1Lk8E9OwgZTTVz4bBOer7JuazaBa+xTkM895tDiWc= github.com/secure-systems-lab/go-securesystemslib v0.10.0 h1:l+H5ErcW0PAehBNrBxoGv1jjNpGYdZ9RcheFkB2WI14= github.com/secure-systems-lab/go-securesystemslib v0.10.0/go.mod h1:MRKONWmRoFzPNQ9USRF9i1mc7MvAVvF1LlW8X5VWDvk= @@ -2917,6 +2920,7 @@ k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kms v0.35.0/go.mod h1:VT+4ekZAdrZDMgShK37vvlyHUVhwI9t/9tvh0AyCWmQ= k8s.io/kms v0.37.0 h1:WDW3pSc5yFB1Od4YXXL3i5oK+vXI2v++LAV82+Ks1Z0= k8s.io/kms v0.37.0/go.mod h1:0U6zfwLkfJr9O/jLgz2GARnKLh2uIhkq2EeaU66J5ow= +k8s.io/kms v0.37.1 h1:MD1kA8AdFF/N5sGP8gnvL6vTo/oeaQXbmLjWmmJHLc8= k8s.io/kms v0.37.1/go.mod h1:0U6zfwLkfJr9O/jLgz2GARnKLh2uIhkq2EeaU66J5ow= k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= diff --git a/pkg/config/config.go b/pkg/config/config.go index 473ecf268f..20812aed9f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -122,6 +122,8 @@ type Config struct { EnableMetricsExporter bool `mapstructure:"prometheusExporterEnabled"` EnableSandboxPolicy bool `mapstructure:"sandboxPolicyEnabled"` SandboxPolicyImage string `mapstructure:"sandboxPolicyImage"` + SandboxPolicyPrivilegeImage string `mapstructure:"sandboxPolicyPrivilegeImage"` + SandboxPolicyQualifiedKernels []string `mapstructure:"sandboxPolicyQualifiedKernels"` SandboxPolicyContainerdSocket string `mapstructure:"sandboxPolicyContainerdSocket"` SandboxPolicyControlRoot string `mapstructure:"sandboxPolicyControlRoot"` SandboxPolicyMaxOwners uint64 `mapstructure:"sandboxPolicyMaxOwners"` @@ -226,6 +228,9 @@ func LoadConfigOptional(path string, errNotFound bool) (Config, error) { viper.SetDefault("sandboxPolicyEnabled", false) viper.SetDefault("sandboxPolicyImage", "") + viper.SetDefault("sandboxPolicyPrivilegeImage", "") + // Unset: the built-in qualified kernel list. ["*"] admits every kernel. + viper.SetDefault("sandboxPolicyQualifiedKernels", nil) viper.SetDefault("sandboxPolicyContainerdSocket", "") viper.SetDefault("sandboxPolicyControlRoot", "") viper.SetDefault("sandboxPolicyMaxOwners", 4) diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings.go b/pkg/ebpf/gadgets/common/enforcement/bindings.go index 6a6a414489..976bf11388 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings.go @@ -33,6 +33,11 @@ type ContainerIdentity = execruntime.ContainerIdentity type Binding struct { Container ContainerIdentity Slot uint32 + // SkipArming binds without arming the tasks already in the cgroup. Use it + // for a parent cgroup bound before its containers start: a container + // runtime helper can be there between its namespace setup and its exec, + // and must not be judged. Tasks are still armed at their own exec. + SkipArming bool // Subtree also covers the descendant cgroups. Scope is exact by default; // set it only for a parent cgroup bound before its containers start, and // report it in status. @@ -302,8 +307,8 @@ func (b *Bindings) bind(c Binding) error { } // A slot move arms again too: a task that the old slot left unarmed (an // audit slot allows a fork whose child mark failed) must not escape the - // new slot. - if b.armed != nil { + // new slot. A SkipArming binding never arms. + if b.armed != nil && !c.SkipArming { if err := b.armExisting(b.armed, id.CgroupPath); err != nil { if bound && !errors.Is(err, ErrContainerGone) { // Undo the move, so a retry moves and arms again. diff --git a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go index c30e9ec90a..df9cf5c64e 100644 --- a/pkg/ebpf/gadgets/common/enforcement/bindings_test.go +++ b/pkg/ebpf/gadgets/common/enforcement/bindings_test.go @@ -247,6 +247,31 @@ func TestSlotFaults(t *testing.T) { } } +func TestSkipArmingBindsWithoutArming(t *testing.T) { + b, policy, members, _ := newFixture(t, func(Map) error { return nil }) + var armedPaths []string + b.armExisting = func(_ Map, path string) error { armedPaths = append(armedPaths, path); return nil } + parent := ContainerIdentity{Runtime: "parent", ID: "p1", CgroupID: 7, CgroupPath: "/kubepods.slice/kubepods-pod1.slice"} + b.desired[selector(parent)] = Binding{Container: parent, SkipArming: true} + if err := b.start(policy, members, &fakeMap{typ: ebpf.TaskStorage}, nil, nil); err != nil { + t.Fatal(err) + } + // Only the initial container binding arms; the replayed parent does not. + if len(armedPaths) != 1 || armedPaths[0] != identity.CgroupPath || members.data[uint64(7)] != (bindingValue{}) { + t.Fatalf("startup armed %v bindings %v", armedPaths, members.data) + } + if err := b.Unbind(parent); err != nil { + t.Fatal(err) + } + if err := b.Bind(Binding{Container: parent, Slot: 1, SkipArming: true}); err != nil || len(armedPaths) != 1 || members.data[uint64(7)] != (bindingValue{Slot: 1}) { + t.Fatalf("live bind: %v armed %v bindings %v", err, armedPaths, members.data) + } + // A slot move of a SkipArming binding does not arm either. + if err := b.Bind(Binding{Container: parent, Slot: 0, SkipArming: true}); err != nil || len(armedPaths) != 1 { + t.Fatalf("slot move: %v armed %v", err, armedPaths) + } +} + func TestBindingFlags(t *testing.T) { log := &[]string{} policy := &fakeMap{typ: ebpf.Hash, data: map[interface{}]interface{}{}, log: log, name: "policy"} diff --git a/pkg/ebpf/gadgets/enforce_privilege/README.md b/pkg/ebpf/gadgets/enforce_privilege/README.md index 2402ff9c8d..9a05504bb7 100644 --- a/pkg/ebpf/gadgets/enforce_privilege/README.md +++ b/pkg/ebpf/gadgets/enforce_privilege/README.md @@ -135,7 +135,10 @@ policy: - **Scope.** A binding covers its exact cgroup, as in enforce_exec. A binding with `Subtree` (`BINDING_SUBTREE`) also covers its descendant cgroups at any depth; the owner sets it only for a parent cgroup bound before its containers - start, and reports it. A subtree binding must be at cgroup level 31 or less + start, and reports it. With `SkipArming` a container is then judged from its + first exec; the tasks already in the parent are not armed, also not at a + slot move or in a new instance, because a runtime helper can be there during + its setup. A subtree binding must be at cgroup level 31 or less (the controller refuses a deeper one; levels are counted in the owner's cgroup namespace, the kernel counts from the host root). The task's own binding wins, then the nearest parent that is a subtree binding or a Strict @@ -157,6 +160,12 @@ policy: ## Known limits +- **Parent bindings.** A subtree parent binding also judges the armed tasks + of its child cgroups that have no binding of their own, for example an + infrastructure (pause) process. OCI hooks are not armed under + `RuntimeHelpers` (see the next item). A new instance does not arm the tasks + that run only under a parent binding; they are judged again when their own + cgroup is bound or at their next exec. - **Runtime helpers are trusted under `RuntimeHelpers`.** In a binding with the flag, a task forked by an unarmed task inside the container (an OCI hook, or a process a hook leaves behind) is not judged, and neither are its children. diff --git a/pkg/sandboxpolicy/availability.go b/pkg/sandboxpolicy/availability.go new file mode 100644 index 0000000000..80a77779b3 --- /dev/null +++ b/pkg/sandboxpolicy/availability.go @@ -0,0 +1,45 @@ +package sandboxpolicy + +import ( + "errors" + "fmt" +) + +// EnforcementUnavailableError means a policy section cannot be enforced on this +// node. Reason becomes the Ready condition reason in the node status. +type EnforcementUnavailableError struct { + Section, Reason, Detail string + // Enforcing: the section enforces, but must not report Ready (for + // example on a kernel that is not qualified). + Enforcing bool +} + +func (e *EnforcementUnavailableError) Error() string { + if e.Enforcing { + return fmt.Sprintf("%s enforces on this node but is not qualified: %s", e.Section, e.Detail) + } + return fmt.Sprintf("%s enforcement is unavailable on this node: %s", e.Section, e.Detail) +} + +// PrivilegeAvailability is implemented by a backend that knows whether the +// privilege section can be enforced on this node. +type PrivilegeAvailability interface { + PrivilegeUnavailable() error +} + +// privilegeUnavailable returns why the privilege section cannot be enforced, +// or nil. It is called with c.mu held. +func (c *Controller) privilegeUnavailable() error { + if a, ok := c.backend.(PrivilegeAvailability); ok { + return a.PrivilegeUnavailable() + } + return nil +} + +func unavailableReason(err error) (string, string, bool) { + var u *EnforcementUnavailableError + if !errors.As(err, &u) { + return "", "", false + } + return u.Reason, u.Error(), true +} diff --git a/pkg/sandboxpolicy/compiler.go b/pkg/sandboxpolicy/compiler.go index bf8d5eae60..371da788be 100644 --- a/pkg/sandboxpolicy/compiler.go +++ b/pkg/sandboxpolicy/compiler.go @@ -30,8 +30,8 @@ func Compile(ctx context.Context, identity PolicyIdentity, spec api.PolicySpec, if spec.Mode != api.ModeAudit && spec.Mode != api.ModeEnforce { return result, fmt.Errorf("policy is not active: %q", spec.Mode) } - if spec.File != nil || spec.Network != nil || spec.Privilege != nil || spec.Identity != nil { - return result, fmt.Errorf("non-exec sections are unsupported") + if spec.File != nil || spec.Network != nil || spec.Identity != nil { + return result, fmt.Errorf("file, network and identity sections are unsupported") } // JSON cloning preserves all pointers and slice fields of the API vocabulary. data, err := json.Marshal(spec) @@ -52,6 +52,7 @@ func Compile(ctx context.Context, identity PolicyIdentity, spec api.PolicySpec, return result, fmt.Errorf("exec defaultMode requires native IG ABI support") } } + result.Privilege = copy.Privilege names := make(map[string]bool) for _, r := range result.Exec.Rules { names[r.Name] = true @@ -127,7 +128,7 @@ func Compile(ctx context.Context, identity PolicyIdentity, spec api.PolicySpec, } } // Validate imported rules as well as authored rules after class expansion. - validated := api.PolicySpec{Containers: copy.Containers, Mode: copy.Mode, RootRole: result.RootRole, Exec: &result.Exec} + validated := api.PolicySpec{Containers: copy.Containers, Mode: copy.Mode, RootRole: result.RootRole, Exec: &result.Exec, Privilege: result.Privilege} if err := api.ValidatePolicySpec(validated); err != nil { return result, err } @@ -173,8 +174,13 @@ func validateShape(spec api.PolicySpec) error { if spec.Mode == api.ModeOff { return nil } - if spec.File != nil || spec.Network != nil || spec.Privilege != nil || spec.Identity != nil { - return fmt.Errorf("non-exec sections are unsupported") + if spec.File != nil || spec.Network != nil || spec.Identity != nil { + return fmt.Errorf("file, network and identity sections are unsupported") + } + if spec.Privilege != nil { + if _, err := PrivilegeDocument("", spec.Mode, *spec.Privilege); err != nil { + return fmt.Errorf("privilege: %w", err) + } } if spec.Exec != nil { if spec.Exec.Default != api.ActionAllow && spec.Exec.Default != api.ActionDeny { diff --git a/pkg/sandboxpolicy/controller.go b/pkg/sandboxpolicy/controller.go index 2bac0ab566..7c6868f431 100644 --- a/pkg/sandboxpolicy/controller.go +++ b/pkg/sandboxpolicy/controller.go @@ -16,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" ) @@ -28,6 +29,10 @@ var NodeStatusGVR = api.SchemeGroupVersion.WithResource("sandboxpolicynodestatus type Options struct { Node api.NodeReference AgentNamespace string + // EnforcementUnavailable says why no section can enforce or audit on this + // node (for example, BPF LSM is not active). It comes from a probe at startup; + // nil means the probe passed. + EnforcementUnavailable error } type installed struct { @@ -51,6 +56,9 @@ type Controller struct { lastError error bindingError error configError error + // classes and classErrors are from the last reconcile (see PodPlans). + classes api.PolicyClasses + classErrors map[string]error } var _ watcher.Adaptor = (*Controller)(nil) @@ -168,6 +176,209 @@ func (c *Controller) Reconcile(ctx context.Context) error { c.lastError = c.reconcile(ctx) return c.lastError } + +// PrepareScheduled passes, for every container of the given pods (scheduled to +// this node and not finished), the privilege policies it will get to +// a backend that can prepare them. It is a hint and changes no binding state. +func (c *Controller) PrepareScheduled(ctx context.Context, containers []Binding) error { + c.mu.Lock() + preparer, ok := c.backend.(SlotPreparer) + if !ok || c.configError != nil { + c.mu.Unlock() + return nil + } + plans := privilegePlans(c.privilegePolicies(), containers) + c.mu.Unlock() + var nonEmpty []Plan + for _, plan := range plans { + if len(plan.Policies) > 0 { + nonEmpty = append(nonEmpty, plan) + } + } + // Outside c.mu: a preparation may start a gadget instance. + return preparer.Prepare(ctx, nonEmpty) +} + +// PodPlans returns the privilege plan of each pod whose declared containers +// (all of them must be given) get the same nonempty set of privilege policies. +// Such a pod can be bound at its pod cgroup before its containers start. A pod +// whose containers get different policies, or none, has no pod plan: a pod +// binding would also judge the containers that the policies do not select. +// A container must also pass the checks that its own binding needs (see +// podContainerPlan), so a pod binding never applies a policy that the +// container binding refuses. A pod whose input is incomplete (an invalid policy +// or class, a policy that does not compile, too many policies) gets an empty +// plan: keep its current pod binding, as a container binding keeps its last +// good plan. Only an authoritative result (containers with different policies, +// no privilege policy, a policy that needs a ContainerProfile) removes it. It +// returns false when the policies are not known (a configuration error). +func (c *Controller) PodPlans(containers []Binding) (map[types.UID]Plan, bool) { + c.mu.Lock() + if c.configError != nil { + c.mu.Unlock() + return nil, false + } + var policies []desiredPolicy + for _, obj := range c.objects { + if obj.GetKind() != "SandboxPolicyClasses" { + policies = append(policies, parsePolicy(obj)) + } + } + classes, classErrors := c.classes, c.classErrors + c.mu.Unlock() + pods := map[types.UID]Plan{} + mixed := map[types.UID]bool{} + keep := map[types.UID]bool{} + for _, b := range containers { + plan, state := podContainerPlan(policies, classes, classErrors, b) + if state == podPlanKeep { + keep[b.PodUID] = true + } + if mixed[b.PodUID] { + continue + } + if state != podPlanReady { + mixed[b.PodUID] = true + delete(pods, b.PodUID) + continue + } + previous, seen := pods[b.PodUID] + if seen && !samePolicies(previous, plan) { + mixed[b.PodUID] = true + delete(pods, b.PodUID) + continue + } + pods[b.PodUID] = plan + } + for uid := range keep { + pods[uid] = Plan{} // incomplete input: keep the current binding + } + return pods, true +} + +type podPlanState int + +const ( + podPlanNone podPlanState = iota // no pod binding (authoritative) + podPlanReady // the pod plan + podPlanKeep // incomplete input: keep the current binding +) + +// podContainerPlan returns the privilege plan of a container for a pod +// binding. It is false when the container's own binding would fail, as in +// reconcile: an invalid policy that may select it, a class error, too many +// policies, or a policy that does not compile without a ContainerProfile +// resolver (a policy with sources depends on the profile, so it is refused). +func podContainerPlan(policies []desiredPolicy, classes api.PolicyClasses, classErrors map[string]error, b Binding) (Plan, podPlanState) { + var plan Plan + selected := 0 + for _, p := range policies { + if p.err != nil { + if p.identity.Kind == "SandboxPolicy" && p.identity.Namespace != b.Namespace { + continue + } + return Plan{}, podPlanKeep + } + if p.spec.Mode == api.ModeOff || !p.matches(b) { + continue + } + selected++ + if len(p.spec.Sources) > 0 { + // Needs the ContainerProfile, which only the container binding + // resolves. + return Plan{}, podPlanNone + } + if policyClassError(p, classErrors) != nil { + return Plan{}, podPlanKeep + } + if _, err := Compile(context.Background(), p.identity, p.spec, classes, b, nil); err != nil { + return Plan{}, podPlanKeep + } + if p.spec.Privilege != nil { + plan.Policies = append(plan.Policies, CompiledPolicy{Identity: p.identity, Mode: p.spec.Mode, Privilege: p.spec.Privilege.DeepCopy()}) + } + } + if selected > MaxPoliciesPerBinding { + return Plan{}, podPlanKeep + } + if len(plan.Policies) == 0 { + return Plan{}, podPlanNone + } + return plan, podPlanReady +} + +// policyClassError returns the errors of the shared path classes that p's exec +// rules use, as reconcile does (a class that p defines itself is not shared). +func policyClassError(p desiredPolicy, classErrors map[string]error) error { + if p.spec.Exec == nil { + return nil + } + var err error + for _, rule := range p.spec.Exec.Rules { + for _, name := range rule.Target.PathClass { + if p.spec.Classes != nil { + if _, local := p.spec.Classes.PathClasses[name]; local { + continue + } + } + err = errors.Join(err, classErrors[name]) + } + } + return err +} + +// privilegePlans returns the plan of each container, in order (possibly empty). +func privilegePlans(policies []desiredPolicy, containers []Binding) []Plan { + plans := make([]Plan, len(containers)) + for i, b := range containers { + for _, p := range policies { + if p.matches(b) { + plans[i].Policies = append(plans[i].Policies, CompiledPolicy{Identity: p.identity, Mode: p.spec.Mode, Privilege: p.spec.Privilege.DeepCopy()}) + } + } + } + return plans +} + +// samePolicies compares the policy objects and generations of two plans. +func samePolicies(a, b Plan) bool { + if len(a.Policies) != len(b.Policies) { + return false + } + ids := map[PolicyIdentity]int{} + for _, p := range a.Policies { + ids[p.Identity]++ + } + for _, p := range b.Policies { + if ids[p.Identity] == 0 { + return false + } + ids[p.Identity]-- + } + return true +} + +// HasPrivilegePolicies reports whether any active policy has a privilege section. +func (c *Controller) HasPrivilegePolicies() bool { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.privilegePolicies()) > 0 +} + +// privilegePolicies is called with c.mu held. +func (c *Controller) privilegePolicies() []desiredPolicy { + var policies []desiredPolicy + for _, obj := range c.objects { + if obj.GetKind() == "SandboxPolicyClasses" { + continue + } + if p := parsePolicy(obj); p.err == nil && p.spec.Privilege != nil && p.spec.Mode != api.ModeOff { + policies = append(policies, p) + } + } + return policies +} + func (c *Controller) LastError() error { c.mu.Lock(); defer c.mu.Unlock(); return c.lastError } func bindingKey(b Binding) string { @@ -323,6 +534,7 @@ func (c *Controller) reconcile(ctx context.Context) error { classes.PathClasses[name] = paths } } + c.classes, c.classErrors = classes, classErrors policyErrors := make(map[string]error) for _, p := range policies { if p.err != nil { diff --git a/pkg/sandboxpolicy/controller_test.go b/pkg/sandboxpolicy/controller_test.go index 5c1f234d8e..5f06cbcc35 100644 --- a/pkg/sandboxpolicy/controller_test.go +++ b/pkg/sandboxpolicy/controller_test.go @@ -179,6 +179,12 @@ func TestInvalidRevisionRetainsLastGoodWhileClusterUpdates(t *testing.T) { if stale.Conditions[0].Status != metav1.ConditionFalse || !stale.Stale || len(stale.Conditions) != 2 || !strings.Contains(stale.Conditions[1].Message, "generation=1") { t.Fatalf("last-good status missing: %+v", stale) } + if stale.ObservedGeneration != 2 || stale.AppliedGeneration != 1 { + t.Fatalf("applied generation must stay on last-good: %+v", stale) + } + if applied := writer.reports["cluster"].Status.AppliedGeneration; applied != 2 { + t.Fatalf("cluster applied generation %d, want 2", applied) + } } func TestClassErrorOnlyBlocksDependentPolicy(t *testing.T) { @@ -342,6 +348,9 @@ func TestInitialSyncUpdateLastGoodAndWithdrawal(t *testing.T) { if report.Status.Conditions[0].Status != metav1.ConditionFalse || report.Status.ObservedGeneration != 2 || !report.Status.Stale || !strings.Contains(report.Status.Conditions[0].Message, "last-good") { t.Fatalf("misleading report: %+v", report.Status) } + if report.Status.AppliedGeneration != 1 { + t.Fatalf("applied generation %d, want last-good 1", report.Status.AppliedGeneration) + } b.fail = false if err := c.Reconcile(ctx); err != nil { t.Fatal(err) diff --git a/pkg/sandboxpolicy/igbackend/identity_linux.go b/pkg/sandboxpolicy/igbackend/identity_linux.go index f8444017a2..546dba8672 100644 --- a/pkg/sandboxpolicy/igbackend/identity_linux.go +++ b/pkg/sandboxpolicy/igbackend/identity_linux.go @@ -8,6 +8,7 @@ import ( "os" "path" "path/filepath" + "runtime" "strconv" "strings" @@ -21,8 +22,13 @@ import ( type LinuxValidator struct{ ProcRoot, CgroupRoot string } type linuxLease struct { proc, cgroup, namespace *os.File - identity sandboxpolicy.Binding - start string + // hostCgroupNS is the cgroup namespace of PID 1 of procRoot when it is not + // ours. /proc//cgroup is relative to the reader's cgroup namespace, + // and the binding's path is relative to PID 1's (as the container + // collection reads it), so Check reads it there. + hostCgroupNS *os.File + identity sandboxpolicy.Binding + start string } func (v LinuxValidator) Acquire(ctx context.Context, b sandboxpolicy.Binding) (Lease, error) { @@ -89,6 +95,9 @@ func (v LinuxValidator) Acquire(ctx context.Context, b sandboxpolicy.Binding) (L if stat.Ino != b.MountNamespace { return fail(fmt.Errorf("mount namespace changed")) } + if lease.hostCgroupNS, err = foreignCgroupNamespace(procRoot); err != nil { + return fail(err) + } lease.start, err = lease.readStart() if err != nil { return fail(err) @@ -139,7 +148,7 @@ func (l *linuxLease) Check() (err error) { if start != l.start { return fmt.Errorf("process identity changed") } - data, err := l.read("cgroup") + data, err := l.readCgroup() if err != nil { return err } @@ -171,11 +180,59 @@ func (l *linuxLease) Check() (err error) { } func (l *linuxLease) Close() error { var errs []error - for _, f := range []*os.File{l.namespace, l.cgroup, l.proc} { + for _, f := range []*os.File{l.hostCgroupNS, l.namespace, l.cgroup, l.proc} { if f != nil { errs = append(errs, f.Close()) } } - l.namespace, l.cgroup, l.proc = nil, nil, nil + l.hostCgroupNS, l.namespace, l.cgroup, l.proc = nil, nil, nil, nil return errors.Join(errs...) } + +// foreignCgroupNamespace opens the cgroup namespace of PID 1 of procRoot. It +// returns nil when this process is already in it. node-agent normally runs in +// its own (private) cgroup namespace, where /proc//cgroup of a workload +// shows a "/../.." path instead of the path the container collection recorded. +func foreignCgroupNamespace(procRoot string) (*os.File, error) { + host, err := os.Open(filepath.Join(procRoot, "1", "ns", "cgroup")) + if err != nil { + return nil, fmt.Errorf("host cgroup namespace: %w", err) + } + var hostStat, selfStat unix.Stat_t + if err := unix.Fstat(int(host.Fd()), &hostStat); err != nil { + return nil, errors.Join(err, host.Close()) + } + if err := unix.Stat("/proc/thread-self/ns/cgroup", &selfStat); err != nil { + return nil, errors.Join(err, host.Close()) + } + if hostStat.Dev == selfStat.Dev && hostStat.Ino == selfStat.Ino { + return nil, host.Close() + } + return host, nil +} + +// readCgroup reads /proc//cgroup in the host cgroup namespace. +func (l *linuxLease) readCgroup() ([]byte, error) { + if l.hostCgroupNS == nil { + return l.read("cgroup") + } + runtime.LockOSThread() + own, err := os.Open("/proc/thread-self/ns/cgroup") + if err != nil { + runtime.UnlockOSThread() + return nil, err + } + defer own.Close() + if err := unix.Setns(int(l.hostCgroupNS.Fd()), unix.CLONE_NEWCGROUP); err != nil { + runtime.UnlockOSThread() + return nil, fmt.Errorf("enter host cgroup namespace: %w", err) + } + data, readErr := l.read("cgroup") + if err := unix.Setns(int(own.Fd()), unix.CLONE_NEWCGROUP); err != nil { + // Keep the thread locked: Go ends it when this goroutine exits, so no + // other goroutine runs in the wrong cgroup namespace. + return nil, fmt.Errorf("restore cgroup namespace: %w", err) + } + runtime.UnlockOSThread() + return data, readErr +} diff --git a/pkg/sandboxpolicy/igbackend/identity_linux_test.go b/pkg/sandboxpolicy/igbackend/identity_linux_test.go index 9190779fc2..b31421fc82 100644 --- a/pkg/sandboxpolicy/igbackend/identity_linux_test.go +++ b/pkg/sandboxpolicy/igbackend/identity_linux_test.go @@ -143,3 +143,30 @@ func TestLinuxLeaseRejectsCgroupMovementBeforeNamespaceAccess(t *testing.T) { }) } } + +func TestForeignCgroupNamespace(t *testing.T) { + link := func(target string) string { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "1", "ns"), 0700); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, filepath.Join(root, "1", "ns", "cgroup")); err != nil { + t.Fatal(err) + } + return root + } + // In our own cgroup namespace there is nothing to enter. + if ns, err := foreignCgroupNamespace(link("/proc/thread-self/ns/cgroup")); err != nil || ns != nil { + t.Fatalf("own namespace: ns=%v err=%v", ns, err) + } + // A different namespace is kept open for Check. + ns, err := foreignCgroupNamespace(link("/proc/thread-self/ns/mnt")) + if err != nil || ns == nil { + t.Fatalf("foreign namespace: ns=%v err=%v", ns, err) + } + ns.Close() + // A missing host namespace fails closed. + if _, err := foreignCgroupNamespace(filepath.Join(t.TempDir(), "missing")); err == nil { + t.Fatal("missing host cgroup namespace accepted") + } +} diff --git a/pkg/sandboxpolicy/igbackend/kernel.go b/pkg/sandboxpolicy/igbackend/kernel.go new file mode 100644 index 0000000000..3ef505f577 --- /dev/null +++ b/pkg/sandboxpolicy/igbackend/kernel.go @@ -0,0 +1,44 @@ +package igbackend + +import ( + "fmt" + "runtime" + "slices" + + "github.com/kubescape/node-agent/pkg/sandboxpolicy" + "golang.org/x/sys/unix" +) + +// QualifiedKernels are the kernel releases (uname -r) on which the section +// kernel tests passed (AL2023 amd64 fleet kernels). 6.18.56-125.164 joins +// after its run. +var QualifiedKernels = []string{ + "6.12.113-138.209.amzn2023.x86_64", +} + +// AnyKernel in a qualified list admits every kernel (test clusters). +const AnyKernel = "*" + +// CheckQualifiedKernel returns an EnforcementUnavailableError with reason +// PrivilegeKernelUnqualified unless release is in qualified (empty: +// QualifiedKernels). The privilege section still enforces on such a kernel, +// but never reports Ready. +func CheckQualifiedKernel(release string, qualified []string) error { + if len(qualified) == 0 { + qualified = QualifiedKernels + } + if slices.Contains(qualified, AnyKernel) || slices.Contains(qualified, release) { + return nil + } + return &sandboxpolicy.EnforcementUnavailableError{Section: "privilege", Reason: "PrivilegeKernelUnqualified", Enforcing: true, + Detail: fmt.Sprintf("kernel %s (%s) is not in the qualified list %v", release, runtime.GOARCH, qualified)} +} + +// KernelRelease returns the running kernel release (uname -r). +func KernelRelease() (string, error) { + var u unix.Utsname + if err := unix.Uname(&u); err != nil { + return "", err + } + return unix.ByteSliceToString(u.Release[:]), nil +} diff --git a/pkg/sandboxpolicy/igbackend/kernel_test.go b/pkg/sandboxpolicy/igbackend/kernel_test.go new file mode 100644 index 0000000000..fcae5153e7 --- /dev/null +++ b/pkg/sandboxpolicy/igbackend/kernel_test.go @@ -0,0 +1,29 @@ +package igbackend + +import ( + "errors" + "strings" + "testing" + + "github.com/kubescape/node-agent/pkg/sandboxpolicy" +) + +func TestCheckQualifiedKernel(t *testing.T) { + if err := CheckQualifiedKernel("6.12.113-138.209.amzn2023.x86_64", nil); err != nil { + t.Fatalf("a fleet kernel is qualified: %v", err) + } + err := CheckQualifiedKernel("6.12.114-139.1.amzn2023.x86_64", nil) + var u *sandboxpolicy.EnforcementUnavailableError + if !errors.As(err, &u) || u.Reason != "PrivilegeKernelUnqualified" { + t.Fatalf("another kernel is not qualified: %v", err) + } + if err := CheckQualifiedKernel("6.17.0-lab", []string{AnyKernel}); err != nil { + t.Fatalf("the wildcard admits every kernel: %v", err) + } + if err := CheckQualifiedKernel("6.12.113-138.209.amzn2023.x86_64", []string{}); err != nil { + t.Fatalf("an empty list (a Helm default) is the built-in list: %v", err) + } + if !u.Enforcing || !strings.Contains(u.Error(), "enforces") { + t.Fatalf("the section still enforces: %q", u.Error()) + } +} diff --git a/pkg/sandboxpolicy/igbackend/lsm.go b/pkg/sandboxpolicy/igbackend/lsm.go new file mode 100644 index 0000000000..d2e4e49126 --- /dev/null +++ b/pkg/sandboxpolicy/igbackend/lsm.go @@ -0,0 +1,35 @@ +package igbackend + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/kubescape/node-agent/pkg/sandboxpolicy" +) + +// CheckBPFLSM returns an EnforcementUnavailableError unless "bpf" is in the +// host's active LSM list. Without it, BPF LSM programs attach but never run +// (for example the default AKS Ubuntu and Azure Linux images), so enforcement +// would be reported while nothing is enforced. Every section enforces through +// BPF LSM, so the result applies to all of them. +func CheckBPFLSM(hostRoot string) error { + path := filepath.Join(hostRoot, "sys/kernel/security/lsm") + data, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return &sandboxpolicy.EnforcementUnavailableError{Section: "sandbox policy", Reason: "BPFLSMInactive", Detail: fmt.Sprintf("%s is missing (securityfs not mounted)", path)} + } + if err != nil { + return &sandboxpolicy.EnforcementUnavailableError{Section: "sandbox policy", Reason: "BPFLSMInactive", Detail: fmt.Sprintf("reading %s: %v", path, err)} + } + for _, lsm := range strings.Split(strings.TrimSpace(string(data)), ",") { + if lsm == "bpf" { + return nil + } + } + return &sandboxpolicy.EnforcementUnavailableError{Section: "sandbox policy", Reason: "BPFLSMInactive", + Detail: fmt.Sprintf("BPF LSM is not active (active LSMs: %s); add bpf to the lsm= kernel parameter", strings.TrimSpace(string(data)))} +} diff --git a/pkg/sandboxpolicy/igbackend/lsm_test.go b/pkg/sandboxpolicy/igbackend/lsm_test.go new file mode 100644 index 0000000000..4a2a4fec2b --- /dev/null +++ b/pkg/sandboxpolicy/igbackend/lsm_test.go @@ -0,0 +1,37 @@ +package igbackend + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/kubescape/node-agent/pkg/sandboxpolicy" +) + +func TestCheckBPFLSM(t *testing.T) { + write := func(list string) string { + root := t.TempDir() + dir := filepath.Join(root, "sys/kernel/security") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "lsm"), []byte(list), 0o644); err != nil { + t.Fatal(err) + } + return root + } + if err := CheckBPFLSM(write("lockdown,capability,landlock,yama,apparmor,bpf\n")); err != nil { + t.Fatalf("bpf active: %v", err) + } + for name, root := range map[string]string{ + "no bpf": write("lockdown,capability,landlock,yama,apparmor\n"), + "bpf-like": write("lockdown,capability,bpfx\n"), + "no securityfs": t.TempDir(), + } { + var u *sandboxpolicy.EnforcementUnavailableError + if err := CheckBPFLSM(root); !errors.As(err, &u) || u.Reason != "BPFLSMInactive" { + t.Errorf("%s: %v", name, err) + } + } +} diff --git a/pkg/sandboxpolicy/igbackend/privilege.go b/pkg/sandboxpolicy/igbackend/privilege.go new file mode 100644 index 0000000000..9bf0e64ca5 --- /dev/null +++ b/pkg/sandboxpolicy/igbackend/privilege.go @@ -0,0 +1,654 @@ +package igbackend + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + "sync" + "time" + + "github.com/kubescape/go-logger" + "github.com/kubescape/go-logger/helpers" + "github.com/kubescape/node-agent/pkg/ebpf/gadgets/common/enforcement" + privpolicy "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/policy" + privruntime "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/runtime" + "github.com/kubescape/node-agent/pkg/sandboxpolicy" + "k8s.io/apimachinery/pkg/types" +) + +// PrivilegeEngine starts one node-wide enforce_privilege instance. Start returns +// only after native readiness, or after failed-candidate cleanup. +type PrivilegeEngine interface { + Start(ctx context.Context, id string, config privruntime.Config) (PrivilegeInstance, error) +} + +type PrivilegeInstance interface { + Ready() error + // SlotHealth fails when the instance counted a tracking fault in the slot: + // a task of an enforcing slot may be untracked, or a fork was refused. + SlotHealth(slot uint32) error + Bind(enforcement.Binding) error + Unbind(enforcement.ContainerIdentity) error + Stop(context.Context) error +} + +// PrivilegeStats counts how container binds found their policy slot. +type PrivilegeStats struct { + // Prepared: the slot already existed (created when the pod was scheduled, + // or used by another container), so the bind was a live map write. + Prepared uint64 + // Replaced: the slot was missing, so the node instance was replaced. + Replaced uint64 + // Replacements counts every node instance replacement. + Replacements uint64 +} + +type privilegeBinding struct { + binding sandboxpolicy.Binding + revision string + combination string + // lease pins the live container identity while it is bound (nil when the + // backend has no validator). + lease Lease + // pendingUnbind: withdrawn while the instance was not ready. Its + // programs may still enforce, so the record stays until the unbind + // succeeds or a new instance (which never binds it) replaces the old one. + pendingUnbind bool +} + +func (p *privilegeBinding) check() error { + if p.lease == nil { + return nil + } + return p.lease.Check() +} + +func (p *privilegeBinding) close() error { + if p.lease == nil { + return nil + } + return p.lease.Close() +} + +// podBinding is a bound pod cgroup (see sandboxpolicy.PodBinding). +type podBinding struct { + pod sandboxpolicy.PodBinding + combination string + // confirmed: bound in the current instance before any container of the + // pod started, so every task of the pod's containers is judged from its + // first exec. + confirmed bool +} + +// podRuntime is the runtime name of a pod cgroup identity in the bindings +// controller. No container runtime has this name. +const podRuntime = "kubernetes-pod" + +// nativePod returns the bindings controller identity of a pod cgroup. It +// covers the container cgroups below it (Subtree) and admits the runtime +// helper exemption for the OCI hooks that run before a container's workload +// (RuntimeHelpers). Tasks already in it are not armed: runc init of a +// container being created can be there. +func nativePod(p sandboxpolicy.PodBinding, slot uint32) enforcement.Binding { + return enforcement.Binding{Container: enforcement.ContainerIdentity{Runtime: podRuntime, ID: string(p.PodUID), CgroupID: p.CgroupID, CgroupPath: p.CgroupPath}, Slot: slot, + Subtree: true, RuntimeHelpers: true, SkipArming: true} +} + +// PrivilegeBackend owns ONE enforce_privilege instance per node. +// +// A slot is one combination of privilege policies; containers with the same +// combination share it. Binding or unbinding a container is a live map write. +// A combination that the running instance does not have (a new or changed +// policy) replaces the node instance: start the candidate with every slot in +// use, wait until it is ready with every container bound, then stop the old +// instance. Both enforce during the overlap. Unused slots are dropped at the +// next replacement. +type PrivilegeBackend struct { + mu sync.Mutex + engine PrivilegeEngine + validator Validator + prefix string + generation uint64 + instance PrivilegeInstance + slots map[string]uint32 // combination -> slot in instance + combinations map[string][]sandboxpolicy.CompiledPolicy + bindings map[identity]*privilegeBinding + pods map[types.UID]*podBinding + prepared map[string][]sandboxpolicy.CompiledPolicy + // reserved is the combination of the last PrepareReplace, kept in every + // replacement until the Replace that follows it consumes it. + reserved map[string][]sandboxpolicy.CompiledPolicy + retiring []PrivilegeInstance + stats PrivilegeStats + closed bool + // emptyFailed is when starting an instance with no slot last failed. + emptyFailed time.Time + // lastHeal is when a tracking fault last started a new instance. + lastHeal time.Time +} + +// emptyRetry is the shortest time between two starts of an instance with no +// slot: each start is a slow gadget load on the refresh loop. +const emptyRetry = 30 * time.Second + +// healRetry is the shortest time between two new instances started only to +// heal tracking faults. Faults recur under memory pressure; a gadget reload on +// every reconcile would add load then. The binding stays unconfirmed meanwhile. +const healRetry = 30 * time.Second + +// nodeHealRetry replaces healRetry when only node faults fail the health: a +// new instance cannot fix a cause that persists, and each reload unconfirms +// the pod bindings of running pods. +const nodeHealRetry = 10 * time.Minute + +// ErrNodeFaultsOnly marks a slot health error caused only by node tracking +// faults (see PrivilegeInstance.SlotHealth). +var ErrNodeFaultsOnly = errors.New("node tracking faults") + +var ( + _ sandboxpolicy.Backend = (*PrivilegeBackend)(nil) + _ sandboxpolicy.ReplacePreparer = (*PrivilegeBackend)(nil) + _ sandboxpolicy.ProtectionAttester = (*PrivilegeBackend)(nil) + _ sandboxpolicy.PodBinder = (*PrivilegeBackend)(nil) +) + +// NewPrivilege creates the node backend. validator verifies the live container +// identity before a bind and holds it while bound; nil skips the check (tests). +func NewPrivilege(engine PrivilegeEngine, validator Validator, prefix string) (*PrivilegeBackend, error) { + if engine == nil { + return nil, fmt.Errorf("privilege engine is required") + } + return &PrivilegeBackend{engine: engine, validator: validator, prefix: prefix, slots: map[string]uint32{}, combinations: map[string][]sandboxpolicy.CompiledPolicy{}, bindings: map[identity]*privilegeBinding{}, pods: map[types.UID]*podBinding{}, prepared: map[string][]sandboxpolicy.CompiledPolicy{}}, nil +} + +// Combination returns the privilege policies of a plan, sorted, and their key. +func Combination(plan sandboxpolicy.Plan) (string, []sandboxpolicy.CompiledPolicy) { + var policies []sandboxpolicy.CompiledPolicy + for _, p := range plan.Policies { + if p.Privilege != nil { + policies = append(policies, p) + } + } + sort.Slice(policies, func(i, j int) bool { return policies[i].Identity.UID < policies[j].Identity.UID }) + type part struct { + UID, Name, Mode string + Privilege any + } + parts := make([]part, len(policies)) + for i, p := range policies { + parts[i] = part{string(p.Identity.UID), p.Identity.Namespace + "/" + p.Identity.Name, string(p.Mode), p.Privilege} + } + data, _ := json.Marshal(parts) // concrete JSON-serializable API structs + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), policies +} + +// logBind records whether a new container found its slot ready ("prepared") +// or needed a node instance replacement ("replaced"). +func logBind(b sandboxpolicy.Binding, slot string) { + logger.L().Info("sandbox privilege bind", helpers.String("slot", slot), + helpers.String("namespace", b.Namespace), helpers.String("pod", b.PodName), helpers.String("container", b.ContainerName)) +} + +func (b *PrivilegeBackend) Stats() PrivilegeStats { + b.mu.Lock() + defer b.mu.Unlock() + return b.stats +} + +func (b *PrivilegeBackend) Replace(ctx context.Context, binding sandboxpolicy.Binding, plan sandboxpolicy.Plan) (sandboxpolicy.ReplaceResult, error) { + b.mu.Lock() + defer b.mu.Unlock() + k := key(binding) + current := b.bindings[k] + var result sandboxpolicy.ReplaceResult + var health error + defer func() { b.reserved = nil }() // consumed, whatever the outcome + if current != nil { + result.ActiveRevision = current.revision + health = b.health(current) + result.ProtectionUnconfirmed = health != nil + } + if b.closed { + return result, fmt.Errorf("privilege backend is closed") + } + combination, policies := Combination(plan) + if len(policies) == 0 { + return result, fmt.Errorf("plan has no privilege section") + } + if b.instance != nil && health == nil { + health = b.instance.Ready() + } + if current != nil && current.revision == plan.Revision && current.combination == combination && health == nil && !current.pendingUnbind { + result.Committed = true + return result, b.retire(ctx) + } + if current != nil && current.revision == plan.Revision && current.combination == combination && !current.pendingUnbind && + b.instance.Ready() == nil && current.check() == nil { + // Only tracking faults: heal with a new instance, at most every + // healRetry. Node faults alone come from any workload on the node + // and may persist, so they heal at most every nodeHealRetry. + retry := healRetry + if errors.Is(health, ErrNodeFaultsOnly) { + retry = nodeHealRetry + } + if time.Since(b.lastHeal) < retry { + return result, health + } + b.lastHeal = time.Now() + } + next := &privilegeBinding{binding: binding, revision: plan.Revision, combination: combination} + if current != nil && current.lease != nil && current.check() == nil { + next.lease = current.lease + } else if b.validator != nil { + lease, err := b.validator.Acquire(ctx, binding) + if err != nil { + return result, err + } + next.lease = lease + } + commit := func() (sandboxpolicy.ReplaceResult, error) { + var errs []error + if current != nil && current.lease != nil && current.lease != next.lease { + errs = append(errs, current.close()) + } + errs = append(errs, b.retire(ctx)) + return sandboxpolicy.ReplaceResult{Committed: true, ActiveRevision: plan.Revision}, errors.Join(errs...) + } + fail := func(err error) (sandboxpolicy.ReplaceResult, error) { + if next.lease != nil && (current == nil || next.lease != current.lease) { + err = errors.Join(err, next.close()) + } + return result, err + } + if slot, ok := b.slots[combination]; ok && b.instance != nil && health == nil { + if err := b.instance.Bind(enforcement.Binding{Container: native(binding), Slot: slot}); err != nil { + return fail(err) + } + if current == nil { + b.stats.Prepared++ + logBind(binding, "prepared") + } + b.bindings[k] = next + return commit() + } + extra := map[string][]sandboxpolicy.CompiledPolicy{combination: policies} + err := b.replaceInstance(ctx, b.prepared, extra, k, next, true) + if err != nil && len(b.prepared) > 0 { + // The speculative prepared slots may not fit: drop only those. The + // active pod bindings stay; if the candidate still does not fit, the + // new binding is refused. + err = b.replaceInstance(ctx, nil, extra, k, next, true) + } + if err != nil { + return fail(err) + } + if current == nil { + b.stats.Replaced++ + logBind(binding, "replaced") + } + return commit() +} + +// PrepareReplace makes sure the plan's combination has a slot in a ready +// instance, replacing the node instance if needed, and binds nothing. The slot +// stays reserved in later replacements until a Replace consumes it, so the +// Replace that follows is normally a live map write. +func (b *PrivilegeBackend) PrepareReplace(ctx context.Context, _ sandboxpolicy.Binding, plan sandboxpolicy.Plan) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return fmt.Errorf("privilege backend is closed") + } + combination, policies := Combination(plan) + if len(policies) == 0 { + return fmt.Errorf("plan has no privilege section") + } + // Reserved only when the slot exists: a combination that does not fit + // must never be carried into other workloads' replacements. + b.reserved = nil + if _, ok := b.slots[combination]; ok && b.instance != nil && b.instance.Ready() == nil { + b.reserved = map[string][]sandboxpolicy.CompiledPolicy{combination: policies} + return nil + } + extra := map[string][]sandboxpolicy.CompiledPolicy{combination: policies} + err := b.replaceInstance(ctx, b.prepared, extra, identity{}, nil, true) + if err != nil && len(b.prepared) > 0 { + err = b.replaceInstance(ctx, nil, extra, identity{}, nil, true) + } + if err == nil { + b.reserved = extra + } + return errors.Join(err, b.retire(ctx)) +} + +func (b *PrivilegeBackend) Withdraw(ctx context.Context, binding sandboxpolicy.Binding) (sandboxpolicy.WithdrawResult, error) { + b.mu.Lock() + defer b.mu.Unlock() + k := key(binding) + current := b.bindings[k] + if current == nil { + return sandboxpolicy.WithdrawResult{Committed: true}, b.retire(ctx) + } + // An instance that is not ready may still run its programs: keep the + // record as pending and let the caller retry. + if b.instance != nil { + if err := b.instance.Ready(); err != nil { + current.pendingUnbind = true + // A new instance never binds a pending record; replace the + // instance that is not ready, at most every healRetry, so a + // pending withdraw cannot wait forever. + if time.Since(b.lastHeal) < healRetry { + return sandboxpolicy.WithdrawResult{ProtectionUnconfirmed: true}, fmt.Errorf("privilege unbind pending: %w", err) + } + b.lastHeal = time.Now() + if rerr := b.replaceInstance(ctx, b.prepared, nil, identity{}, nil, true); rerr != nil { + return sandboxpolicy.WithdrawResult{ProtectionUnconfirmed: true}, errors.Join(fmt.Errorf("privilege unbind pending: %w", err), rerr) + } + // replaceInstance dropped the record and closed its lease. + return sandboxpolicy.WithdrawResult{Committed: true}, b.retire(ctx) + } + if err := b.instance.Unbind(native(binding)); err != nil { + current.pendingUnbind = true + return sandboxpolicy.WithdrawResult{ProtectionUnconfirmed: true}, err + } + } + delete(b.bindings, k) + return sandboxpolicy.WithdrawResult{Committed: true}, errors.Join(current.close(), b.retire(ctx)) +} + +func (b *PrivilegeBackend) Attest(_ context.Context, binding sandboxpolicy.Binding, revision string) error { + b.mu.Lock() + defer b.mu.Unlock() + current := b.bindings[key(binding)] + if current == nil || current.revision != revision || b.instance == nil { + return fmt.Errorf("privilege revision %q is not active", revision) + } + return b.health(current) +} + +// health checks the instance, the slot's tracking faults and the identity +// lease of one bound container. A fault makes the next same-plan Replace start +// a new instance, which arms the running tasks again. +func (b *PrivilegeBackend) health(p *privilegeBinding) error { + slot, ok := b.slots[p.combination] + if !ok { + return errors.Join(fmt.Errorf("privilege combination has no slot"), b.instance.Ready(), p.check()) + } + return errors.Join(b.instance.Ready(), b.instance.SlotHealth(slot), p.check()) +} + +// Prepare declares the combinations of the pods scheduled to this node that are +// not finished, so containers that start later find their slot. A missing combination replaces the node +// instance now, so the containers later bind with a live map write. +// Combinations no longer prepared and not in use are dropped at the next +// replacement. +func (b *PrivilegeBackend) Prepare(ctx context.Context, plans []sandboxpolicy.Plan) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return fmt.Errorf("privilege backend is closed") + } + prepared := map[string][]sandboxpolicy.CompiledPolicy{} + // Without an instance, start one even with no plans: the first gadget load + // is slow, so the first container of a privilege policy would find no + // instance ready (for example the first pod after a node-agent restart). + missing := b.instance == nil && time.Since(b.emptyFailed) >= emptyRetry + // An instance that is not ready protects nothing that can be confirmed, + // and it holds pending withdrawals: replace it, at most every healRetry. + if b.instance != nil && b.instance.Ready() != nil && time.Since(b.lastHeal) >= healRetry { + b.lastHeal = time.Now() + missing = true + } + for _, plan := range plans { + combination, policies := Combination(plan) + if len(policies) == 0 { + continue + } + prepared[combination] = policies + if _, ok := b.slots[combination]; !ok { + missing = true + } + } + if !missing { + b.prepared = prepared + return b.retire(ctx) + } + // The prepared set is kept only when the instance with it is running. + err := b.replaceInstance(ctx, prepared, nil, identity{}, nil, true) + if err != nil && len(prepared) == 0 { + b.emptyFailed = time.Now() + } + return errors.Join(err, b.retire(ctx)) +} + +// BindPods binds the given pod cgroups to the slot of their plan and unbinds +// the pod cgroups that are not given (an authoritative removal). A pod whose +// slot does not exist yet is skipped: Prepare creates the slot, and the next +// call binds the pod. A pod whose plan changes keeps its last-good binding +// until the new slot is ready, then moves to it in one map write. A pod given +// with an empty plan (incomplete input, see Controller.PodPlans) keeps its +// current binding. +func (b *PrivilegeBackend) BindPods(ctx context.Context, pods []sandboxpolicy.PodBinding) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return fmt.Errorf("privilege backend is closed") + } + ready := b.instance != nil && b.instance.Ready() == nil + wanted := make(map[types.UID]sandboxpolicy.PodBinding, len(pods)) + for _, p := range pods { + wanted[p.PodUID] = p + } + var errs []error + for uid, current := range b.pods { + next, ok := wanted[uid] + if ok && next.CgroupID == current.pod.CgroupID && next.CgroupPath == current.pod.CgroupPath { + continue // the same pod cgroup: kept, or moved below + } + // Not wanted, or a new cgroup for the pod: remove the old identity. + // An instance that is not ready protects nothing. + if ready { + if err := b.instance.Unbind(nativePod(current.pod, 0).Container); err != nil { + errs = append(errs, err) + continue + } + } + delete(b.pods, uid) + } + for uid, p := range wanted { + combination, policies := Combination(p.Plan) + current := b.pods[uid] + if len(policies) == 0 { + continue + } + if current != nil && current.combination == combination { + // The same enforcement: keep the binding, its cgroup identity + // and its confirmation; record the newer plan (its generation) + // and the containers' state for status. + current.pod.Plan = p.Plan + current.pod.ContainersStarted = current.pod.ContainersStarted || p.ContainersStarted + continue + } + // A pod binding arms no running task: a pod whose containers + // started gets no new pod binding and relies on its container + // bindings. A bound pod whose plan changed moves to the new slot, + // so a restarted container is never judged by a superseded plan, + // but the move is not confirmed. + if p.ContainersStarted && current == nil { + continue + } + slot, ok := b.slots[combination] + if !ok || !ready { + continue + } + if err := b.instance.Bind(nativePod(p, slot)); err != nil { + errs = append(errs, err) + continue + } + b.pods[uid] = &podBinding{pod: p, combination: combination, confirmed: !p.ContainersStarted} + logger.L().Info("sandbox privilege pod bind", helpers.String("namespace", p.Namespace), helpers.String("pod", p.PodName)) + } + return errors.Join(append(errs, b.retire(ctx))...) +} + +// BoundPodPlans returns the plans of the pod cgroups bound now (see +// sandboxpolicy.PodPlanReporter). While the instance is not ready, every pod +// plan is faulted: its protection cannot be confirmed. +func (b *PrivilegeBackend) BoundPodPlans() (confirmed, faulted []sandboxpolicy.Plan) { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return nil, nil + } + ready := b.instance != nil && b.instance.Ready() == nil + for _, p := range b.pods { + slot, ok := b.slots[p.combination] + switch { + case !ready || !ok || b.instance.SlotHealth(slot) != nil: + faulted = append(faulted, p.pod.Plan) + case p.confirmed: + confirmed = append(confirmed, p.pod.Plan) + } + } + return confirmed, faulted +} + +// replaceInstance starts a candidate with every combination in use or prepared +// (plus extra), binds every container (plus next, under nextKey) and, with +// withPods, every pod cgroup, and only then stops the old instance. Without +// withPods, the pod bindings are dropped. On failure the old instance and +// state stay. +func (b *PrivilegeBackend) replaceInstance(ctx context.Context, prepared, extra map[string][]sandboxpolicy.CompiledPolicy, nextKey identity, next *privilegeBinding, withPods bool) error { + // A new instance arms no task under a pod binding, so a carried pod + // binding is confirmed only while no container of the pod has started. + pods := map[types.UID]*podBinding{} + if withPods { + for uid, p := range b.pods { + pods[uid] = &podBinding{pod: p.pod, combination: p.combination, confirmed: !p.pod.ContainersStarted} + } + } + // A pending unbind is not carried: the new instance never binds it, and + // the old instance stops after the switch. + bindings := map[identity]*privilegeBinding{} + var withdrawn []*privilegeBinding + for k, v := range b.bindings { + if v.pendingUnbind { + // Under nextKey, next replaces the record, and Replace's commit + // closes the old lease when next does not reuse it. + if next == nil || k != nextKey { + withdrawn = append(withdrawn, v) + } + continue + } + bindings[k] = v + } + if next != nil { + bindings[nextKey] = next + } + wanted := map[string][]sandboxpolicy.CompiledPolicy{} + for c, p := range prepared { + wanted[c] = p + } + for c, p := range extra { + wanted[c] = p + } + for c, p := range b.reserved { + wanted[c] = p + } + for _, v := range bindings { + if _, ok := wanted[v.combination]; !ok { + wanted[v.combination] = b.combinations[v.combination] + } + } + for _, p := range pods { + if _, ok := wanted[p.combination]; !ok { + wanted[p.combination] = b.combinations[p.combination] + } + } + keys := make([]string, 0, len(wanted)) + for c := range wanted { + keys = append(keys, c) + } + sort.Strings(keys) + slots := map[string]uint32{} + docs := make([][]privpolicy.Document, len(keys)) + for i, c := range keys { + slots[c] = uint32(i) + for _, p := range wanted[c] { + doc, err := sandboxpolicy.PrivilegeDocument(p.Identity.Namespace+"/"+p.Identity.Name, p.Mode, *p.Privilege) + if err != nil { + return err + } + docs[i] = append(docs[i], doc) + } + } + compiled, err := privpolicy.CompileSlots(docs, b.generation+1) + if err != nil { + return err + } + config := privruntime.Config{Policy: compiled} + for _, v := range bindings { + config.InitialBindings = append(config.InitialBindings, enforcement.Binding{Container: native(v.binding), Slot: slots[v.combination]}) + } + for _, p := range pods { + config.InitialBindings = append(config.InitialBindings, nativePod(p.pod, slots[p.combination])) + } + b.generation++ + candidate, err := b.engine.Start(ctx, fmt.Sprintf("%s-privilege-%d", b.prefix, b.generation), config) + if err != nil { + return err + } + if err = candidate.Ready(); err != nil { + if stopErr := candidate.Stop(ctx); stopErr != nil { + b.retiring = append(b.retiring, candidate) // retried by retire + return errors.Join(err, stopErr) + } + return err + } + if b.instance != nil { + b.retiring = append(b.retiring, b.instance) + } + b.instance, b.slots, b.combinations, b.bindings, b.prepared, b.pods = candidate, slots, wanted, bindings, prepared, pods + b.stats.Replacements++ + var errs []error + for _, v := range withdrawn { + errs = append(errs, v.close()) + } + return errors.Join(errs...) +} + +// retire stops replaced instances and keeps the ones that still fail. +func (b *PrivilegeBackend) retire(ctx context.Context) error { + var kept []PrivilegeInstance + var errs []error + for _, i := range b.retiring { + if err := i.Stop(ctx); err != nil { + kept = append(kept, i) + errs = append(errs, err) + } + } + b.retiring = kept + return errors.Join(errs...) +} + +func (b *PrivilegeBackend) Close(ctx context.Context) error { + b.mu.Lock() + defer b.mu.Unlock() + b.closed = true + if b.instance != nil { + b.retiring = append(b.retiring, b.instance) + b.instance = nil + } + var errs []error + for _, v := range b.bindings { + errs = append(errs, v.close()) + } + b.bindings = map[identity]*privilegeBinding{} + b.pods = map[types.UID]*podBinding{} + return errors.Join(append(errs, b.retire(ctx))...) +} diff --git a/pkg/sandboxpolicy/igbackend/privilege_kernel_test.go b/pkg/sandboxpolicy/igbackend/privilege_kernel_test.go new file mode 100644 index 0000000000..3659695cd7 --- /dev/null +++ b/pkg/sandboxpolicy/igbackend/privilege_kernel_test.go @@ -0,0 +1,144 @@ +//go:build linux && sandboxpolicy_kernel + +package igbackend + +// Opt-in kernel test of the privilege backend with the real gadget image. +// Requires root, BPF LSM and a disposable Docker container started with +// seccomp=unconfined, apparmor=unconfined and --cap-add SYS_ADMIN: +// +// IG_PRIVILEGE_KERNEL_TEST=1 +// IG_PRIVILEGE_IMAGE_ARCHIVE OCI archive from `ig image export` +// IG_PRIVILEGE_IMAGE image name in the archive +// IG_PRIVILEGE_CONTAINER_ID full Docker container ID +// IG_PRIVILEGE_CGROUP_PATH its cgroup v2 path under /sys/fs/cgroup, with a leading "/" (as in /proc//cgroup) +// IG_PRIVILEGE_CONTAINER_ID2, IG_PRIVILEGE_CGROUP_PATH2 a second container +// +// With two containers it also checks the node-wide instance: the second +// container with the same policy is a live bind (no new instance), and a +// second container with another policy replaces the instance while the first +// container stays protected. + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/inspektor-gadget/inspektor-gadget/pkg/operators/simple" + "github.com/inspektor-gadget/inspektor-gadget/pkg/runtime/local" + "github.com/kubescape/node-agent/pkg/sandboxpolicy" + api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" + "golang.org/x/sys/unix" + orasoci "oras.land/oras-go/v2/content/oci" +) + +func TestPrivilegeBackendKernel(t *testing.T) { + if os.Getenv("IG_PRIVILEGE_KERNEL_TEST") != "1" { + t.Skip("explicit disposable kernel fixture opt-in required") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + if err := CheckBPFLSM("/"); err != nil { + t.Fatalf("this test needs an active BPF LSM: %v", err) + } + bindingOf := func(id, cgroupPath string) sandboxpolicy.Binding { + var st unix.Stat_t + if err := unix.Stat(filepath.Join("/sys/fs/cgroup", cgroupPath), &st); err != nil { + t.Fatal(err) + } + return sandboxpolicy.Binding{Runtime: "docker", ContainerID: id, CgroupID: st.Ino, CgroupPath: cgroupPath} + } + id := os.Getenv("IG_PRIVILEGE_CONTAINER_ID") + binding := bindingOf(id, os.Getenv("IG_PRIVILEGE_CGROUP_PATH")) + + store, err := orasoci.NewFromTar(ctx, os.Getenv("IG_PRIVILEGE_IMAGE_ARCHIVE")) + if err != nil { + t.Fatal(err) + } + rt := local.New() + if err = rt.Init(nil); err != nil { + t.Fatal(err) + } + defer rt.Close() + engine, err := NewNativePrivilegeEngine(rt, simple.New("no-kubernetes"), store, os.Getenv("IG_PRIVILEGE_IMAGE")) + if err != nil { + t.Fatal(err) + } + backend, err := NewPrivilege(engine, nil, "kernel-test") + if err != nil { + t.Fatal(err) + } + defer backend.Close(context.Background()) + + unshareIn := func(id string) error { + return exec.CommandContext(ctx, "docker", "exec", id, "unshare", "-m", "true").Run() + } + unshare := func() error { return unshareIn(id) } + plan := func(revision string, action api.Action) sandboxpolicy.Plan { + return sandboxpolicy.Plan{Revision: revision, Policies: []sandboxpolicy.CompiledPolicy{{Mode: api.ModeEnforce, Privilege: &api.PrivilegeSection{ + Default: api.ActionAllow, + Rules: []api.PrivilegeRule{{Rule: api.Rule{Name: "mount-namespace", Action: action}, + Target: api.PrivilegeTarget{Syscall: []string{"unshare"}, Flags: []string{"CLONE_NEWNS"}}}}, + }}}} + } + + if err = unshare(); err != nil { + t.Fatalf("baseline unshare failed: %v", err) + } + // An instance with no slot loads before any container needs it. + if err := backend.Prepare(ctx, nil); err != nil || backend.Stats().Replacements != 1 { + t.Fatalf("empty instance: %v %+v", err, backend.Stats()) + } + if err = unshare(); err != nil { + t.Fatalf("an empty instance must judge nothing: %v", err) + } + if r, err := backend.Replace(ctx, binding, plan("deny", api.ActionDeny)); err != nil || !r.Committed { + t.Fatalf("deny plan: %+v %v", r, err) + } + if err = backend.Attest(ctx, binding, "deny"); err != nil { + t.Fatal(err) + } + if unshare() == nil { + t.Fatal("unshare allowed under the deny plan") + } + if id2 := os.Getenv("IG_PRIVILEGE_CONTAINER_ID2"); id2 != "" { + second := bindingOf(id2, os.Getenv("IG_PRIVILEGE_CGROUP_PATH2")) + if r, err := backend.Replace(ctx, second, plan("deny", api.ActionDeny)); err != nil || !r.Committed { + t.Fatalf("second container, same policy: %+v %v", r, err) + } + if s := backend.Stats(); s.Prepared != 1 || s.Replacements != 2 { + t.Fatalf("same policy must be a live bind: %+v", s) + } + if unshareIn(id2) == nil { + t.Fatal("second container not denied") + } + other := plan("other", api.ActionAllow) + other.Policies[0].Identity.UID = "other" + if r, err := backend.Replace(ctx, second, other); err != nil || !r.Committed { + t.Fatalf("second container, other policy: %+v %v", r, err) + } + if s := backend.Stats(); s.Replacements != 3 { + t.Fatalf("a new combination must replace the instance: %+v", s) + } + if unshareIn(id2) != nil { + t.Fatal("second container must be allowed by its new policy") + } + if unshare() == nil { + t.Fatal("first container lost protection across the replacement") + } + if r, err := backend.Withdraw(ctx, second); err != nil || !r.Committed { + t.Fatalf("withdraw second: %+v %v", r, err) + } + } + if r, err := backend.Replace(ctx, binding, plan("allow", api.ActionAllow)); err != nil || !r.Committed { + t.Fatalf("allow plan: %+v %v", r, err) + } + if err = unshare(); err != nil { + t.Fatalf("unshare denied under the allow plan: %v", err) + } + if r, err := backend.Withdraw(ctx, binding); err != nil || !r.Committed { + t.Fatalf("withdraw: %+v %v", r, err) + } +} diff --git a/pkg/sandboxpolicy/igbackend/privilege_native.go b/pkg/sandboxpolicy/igbackend/privilege_native.go new file mode 100644 index 0000000000..1d20bc5032 --- /dev/null +++ b/pkg/sandboxpolicy/igbackend/privilege_native.go @@ -0,0 +1,167 @@ +package igbackend + +import ( + "context" + "errors" + "fmt" + "strconv" + + "github.com/inspektor-gadget/inspektor-gadget/pkg/datasource" + gadgetcontext "github.com/inspektor-gadget/inspektor-gadget/pkg/gadget-context" + "github.com/inspektor-gadget/inspektor-gadget/pkg/gadget-service/api" + apihelpers "github.com/inspektor-gadget/inspektor-gadget/pkg/gadget-service/api-helpers" + instancemanager "github.com/inspektor-gadget/inspektor-gadget/pkg/gadget-service/instance-manager" + "github.com/inspektor-gadget/inspektor-gadget/pkg/operators" + ocihandler "github.com/inspektor-gadget/inspektor-gadget/pkg/operators/oci-handler" + "github.com/inspektor-gadget/inspektor-gadget/pkg/operators/simple" + "github.com/inspektor-gadget/inspektor-gadget/pkg/runtime" + "github.com/kubescape/go-logger" + "github.com/kubescape/go-logger/helpers" + "github.com/kubescape/node-agent/pkg/ebpf/gadgets/common/enforcement" + privruntime "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/runtime" + "oras.land/oras-go/v2" +) + +// NewNativePrivilegeEngine uses an already initialized runtime, like NewNative. +// Each Start is one node-wide instance. +// Privilege verdicts are logged; exporting them as events is not wired yet. +func NewNativePrivilegeEngine(rt runtime.Runtime, kube operators.DataOperator, store oras.ReadOnlyTarget, image string) (PrivilegeEngine, error) { + if rt == nil || kube == nil || store == nil || image == "" { + return nil, fmt.Errorf("runtime, Kubernetes operator, OCI store and privilege image are required") + } + manager, err := instancemanager.New(rt) + if err != nil { + return nil, err + } + return &nativePrivilegeEngine{manager: manager, kube: kube, store: store, image: image}, nil +} + +type nativePrivilegeEngine struct { + manager *instancemanager.Manager + kube operators.DataOperator + store oras.ReadOnlyTarget + image string +} + +type nativePrivilegeInstance struct { + manager *instancemanager.Manager + id string + bindings *enforcement.Bindings + revision uint64 + strict func(slot uint32) bool +} + +func (i *nativePrivilegeInstance) Ready() error { + status := i.bindings.Status() + if !status.Ready || status.Error != nil || status.Revision != i.revision { + return fmt.Errorf("privilege instance not ready: %+v", status) + } + return nil +} + +// SlotHealth fails on a fault counted in the slot, and, for a Strict slot, on +// a node fault (ErrNodeFaultsOnly): a child of an unconfirmed task can start +// in any cgroup (common/arming.h). An unreadable counter is a fault too. An +// Audit slot ignores node faults by design: Audit coverage is reported, but a +// lost Audit judgement never blocks and never makes a policy NotReady. +func (i *nativePrivilegeInstance) SlotHealth(slot uint32) error { + n, err := i.bindings.SlotFaults(slot) + if err != nil { + return err + } + if n > 0 { + return fmt.Errorf("%d tracking faults in privilege slot %d", n, slot) + } + if i.strict == nil || !i.strict(slot) { + return nil + } + w, err := i.bindings.ArmingWitness() + if err != nil { + return err + } + if w.NodeFaults > 0 { + return fmt.Errorf("%w: %d affect Strict privilege slot %d", ErrNodeFaultsOnly, w.NodeFaults, slot) + } + return nil +} + +func (i *nativePrivilegeInstance) Bind(b enforcement.Binding) error { return i.bindings.Bind(b) } + +func (i *nativePrivilegeInstance) Unbind(identity enforcement.ContainerIdentity) error { + return i.bindings.Unbind(identity) +} + +func (i *nativePrivilegeInstance) Stop(ctx context.Context) error { + err := i.manager.StopGadget(ctx, i.id) + if errors.Is(err, instancemanager.ErrNotFound) { + return nil + } + return err +} + +func (e *nativePrivilegeEngine) Start(ctx context.Context, id string, config privruntime.Config) (PrivilegeInstance, error) { + handler := ocihandler.New() + params := apihelpers.ToParamDescs(handler.GlobalParams()).ToParams() + if err := params.Set("allow-enforcement", "true"); err != nil { + return nil, err + } + // Same trust model as NewNative: the archive is administrator-supplied. + if err := handler.Init(params); err != nil { + return nil, err + } + instance := &nativePrivilegeInstance{manager: e.manager, id: id, revision: config.Policy.Generation, strict: privruntime.StrictSlots(config.Policy)} + names := config.Policy.RuleNames + events := simple.New("sandbox-privilege-verdict", simple.WithPriority(100), simple.OnInit(func(gc operators.GadgetContext) error { + ds := gc.GetDataSources()["verdict"] + if ds == nil { + return fmt.Errorf("missing privilege verdict datasource") + } + var err error + instance.bindings, err = privruntime.GetBindings(gc) + if err != nil { + return err + } + fields := map[string]datasource.FieldAccessor{} + for _, name := range []string{"generation", "cgroup", "deny", "mode", "host_pid", "rule_id", "comm", "hook", "syscall", "capability", "fstype"} { + if fields[name] = ds.GetField(name); fields[name] == nil { + return fmt.Errorf("missing verdict field %q", name) + } + } + return ds.Subscribe(func(ds datasource.DataSource, data datasource.Data) error { + ruleID, _ := fields["rule_id"].Uint32(data) + rule := "" + if int(ruleID) < len(names) { + rule = names[ruleID] + } + mode, _ := fields["mode"].Uint32(data) + pid, _ := fields["host_pid"].Uint32(data) + cgroup, _ := fields["cgroup"].Uint64(data) + generation, _ := fields["generation"].Uint64(data) + capability, _ := fields["capability"].Int32(data) + comm, _ := fields["comm"].String(data) + hook, _ := fields["hook"].String(data) + syscall, _ := fields["syscall"].String(data) + fstype, _ := fields["fstype"].String(data) + logger.L().Info("sandbox privilege verdict", + helpers.String("rule", rule), helpers.Int("mode", int(mode)), helpers.Int("pid", int(pid)), + helpers.String("cgroup", fmt.Sprint(cgroup)), helpers.String("generation", fmt.Sprint(generation)), + helpers.String("comm", comm), helpers.String("hook", hook), helpers.String("syscall", syscall), + helpers.Int("capability", int(capability)), helpers.String("fstype", fstype)) + return nil // Reporting never changes enforcement. + }, 100) + })) + runtimeMntns, err := enforcement.RuntimeMountNamespace() + if err != nil { + return nil, fmt.Errorf("runtime mount namespace: %w", err) + } + desired := &api.GadgetInstance{Id: id, GadgetConfig: &api.GadgetRunRequest{Version: api.VersionGadgetRunProtocol, ImageName: e.image, + ParamValues: map[string]string{"operator.oci.ebpf.host-mntns-id": strconv.FormatUint(runtimeMntns, 10)}}} + err = e.manager.StartGadget(ctx, desired, + gadgetcontext.WithOrasReadonlyTarget(e.store), + gadgetcontext.WithAdditionalDataOperators(handler, e.kube, privruntime.New(), events), + privruntime.WithConfig(config)) + if err != nil { + return nil, err + } + return instance, nil +} diff --git a/pkg/sandboxpolicy/igbackend/privilege_test.go b/pkg/sandboxpolicy/igbackend/privilege_test.go new file mode 100644 index 0000000000..df15e02219 --- /dev/null +++ b/pkg/sandboxpolicy/igbackend/privilege_test.go @@ -0,0 +1,716 @@ +package igbackend + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/kubescape/node-agent/pkg/ebpf/gadgets/common/enforcement" + privruntime "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/runtime" + "github.com/kubescape/node-agent/pkg/sandboxpolicy" + api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" + "k8s.io/apimachinery/pkg/types" +) + +type fakePrivilegeInstance struct { + readyErr, stopErr error + stops int + bound map[uint64]uint32 + faults map[uint32]uint64 +} + +func (i *fakePrivilegeInstance) Ready() error { return i.readyErr } +func (i *fakePrivilegeInstance) SlotHealth(slot uint32) error { + if i.faults[slot] > 0 { + return errors.New("tracking fault") + } + return nil +} +func (i *fakePrivilegeInstance) Bind(b enforcement.Binding) error { + i.bound[b.Container.CgroupID] = b.Slot + return nil +} +func (i *fakePrivilegeInstance) Unbind(c enforcement.ContainerIdentity) error { + delete(i.bound, c.CgroupID) + return nil +} +func (i *fakePrivilegeInstance) Stop(context.Context) error { i.stops++; return i.stopErr } + +type fakePrivilegeEngine struct { + instances []*fakePrivilegeInstance + configs []privruntime.Config + startErr error + failIf func(privruntime.Config) bool +} + +func (e *fakePrivilegeEngine) Start(_ context.Context, _ string, c privruntime.Config) (PrivilegeInstance, error) { + if e.startErr != nil { + return nil, e.startErr + } + if e.failIf != nil && e.failIf(c) { + return nil, errors.New("does not fit") + } + i := &fakePrivilegeInstance{bound: map[uint64]uint32{}} + for _, b := range c.InitialBindings { + i.bound[b.Container.CgroupID] = b.Slot + } + e.instances = append(e.instances, i) + e.configs = append(e.configs, c) + return i, nil +} + +func privBinding(cgroup uint64) sandboxpolicy.Binding { + return sandboxpolicy.Binding{Runtime: "containerd", ContainerID: string(rune('a' + cgroup)), CgroupID: cgroup, CgroupPath: "/kubepods/c"} +} + +func privPolicy(uid string, def api.Action, rules ...api.PrivilegeRule) sandboxpolicy.CompiledPolicy { + return sandboxpolicy.CompiledPolicy{Identity: sandboxpolicy.PolicyIdentity{UID: types.UID(uid), Namespace: "ns", Name: uid}, Mode: api.ModeEnforce, Privilege: &api.PrivilegeSection{Default: def, Rules: rules}} +} + +func privPlan(revision string, policies ...sandboxpolicy.CompiledPolicy) sandboxpolicy.Plan { + return sandboxpolicy.Plan{Revision: revision, Policies: policies} +} + +var denyMount = api.PrivilegeRule{Rule: api.Rule{Name: "no-mount", Action: api.ActionDeny, Origin: ptr("o")}, Target: api.PrivilegeTarget{Syscall: []string{"mount"}}} + +func ptr(s string) *string { return &s } + +var allowMount = api.PrivilegeRule{Rule: api.Rule{Name: "mount-ok", Action: api.ActionAllow}, Target: api.PrivilegeTarget{Syscall: []string{"mount"}}} + +func TestPrivilegeNodeInstanceSharesSlots(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + plan := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + if r, err := b.Replace(ctx, privBinding(1), plan); err != nil || !r.Committed { + t.Fatalf("first: %+v %v", r, err) + } + if r, err := b.Replace(ctx, privBinding(2), plan); err != nil || !r.Committed { + t.Fatalf("second: %+v %v", r, err) + } + if len(e.instances) != 1 || e.instances[0].bound[2] != e.instances[0].bound[1] { + t.Fatalf("same combination must share one slot in one instance: %d %+v", len(e.instances), e.instances[0].bound) + } + if s := b.Stats(); s.Replaced != 1 || s.Prepared != 1 { + t.Fatalf("stats %+v", s) + } + // Same plan again: attestation, no new instance. + if r, err := b.Replace(ctx, privBinding(1), plan); err != nil || !r.Committed || len(e.instances) != 1 { + t.Fatalf("same plan: %+v %v", r, err) + } + if err := b.Attest(ctx, privBinding(1), "r1"); err != nil { + t.Fatal(err) + } + // Withdraw is a live unbind. + if r, err := b.Withdraw(ctx, privBinding(2)); err != nil || !r.Committed { + t.Fatalf("withdraw: %+v %v", r, err) + } + if _, ok := e.instances[0].bound[2]; ok || len(e.instances) != 1 { + t.Fatal("withdraw must unbind in the running instance") + } +} + +func TestPrivilegeNewCombinationReplacesWithOverlap(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + if _, err := b.Replace(ctx, privBinding(1), privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount))); err != nil { + t.Fatal(err) + } + if _, err := b.Replace(ctx, privBinding(2), privPlan("r2", privPolicy("p2", api.ActionDeny))); err != nil { + t.Fatal(err) + } + if len(e.instances) != 2 || e.instances[0].stops != 1 || e.instances[1].stops != 0 { + t.Fatal("the new instance starts before the old one stops") + } + if c := e.configs[1]; c.Policy.Slots != 2 || len(c.InitialBindings) != 2 || c.Policy.Generation <= e.configs[0].Policy.Generation { + t.Fatalf("candidate must carry every slot and binding: %+v", c) + } + // A failed candidate keeps the old instance and state. + e.startErr = errors.New("load failed") + if r, err := b.Replace(ctx, privBinding(3), privPlan("r3", privPolicy("p3", api.ActionDeny))); err == nil || r.Committed { + t.Fatalf("expected failure: %+v", r) + } + if e.instances[1].stops != 0 || b.Attest(ctx, privBinding(1), "r1") != nil { + t.Fatal("old instance must keep running with its bindings") + } +} + +func TestPrivilegePrepareCreatesSlotBeforeContainer(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + plan := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + if err := b.Prepare(ctx, []sandboxpolicy.Plan{plan}); err != nil { + t.Fatal(err) + } + if len(e.instances) != 1 || e.configs[0].Policy.Slots != 1 { + t.Fatalf("prepare must create the slot: %d", len(e.instances)) + } + if _, err := b.Replace(ctx, privBinding(1), plan); err != nil || len(e.instances) != 1 { + t.Fatalf("bind after prepare must be live: %v instances=%d", err, len(e.instances)) + } + if s := b.Stats(); s.Prepared != 1 || s.Replaced != 0 { + t.Fatalf("stats %+v", s) + } + // The same prepare again changes nothing. + if err := b.Prepare(ctx, []sandboxpolicy.Plan{plan}); err != nil || len(e.instances) != 1 { + t.Fatalf("repeat prepare: %v", err) + } +} + +func TestPrivilegeMergesPoliciesInOneSlot(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + plan := privPlan("r1", privPolicy("p1", api.ActionDeny, denyMount), privPolicy("p2", api.ActionAllow, allowMount)) + if _, err := b.Replace(context.Background(), privBinding(1), plan); err != nil { + t.Fatal(err) + } + c := e.configs[0].Policy + names := strings.Join(c.RuleNames, ",") + if c.Slots != 1 || !strings.Contains(names, "ns/p1/no-mount") || !strings.Contains(names, "ns/p2/mount-ok") { + t.Fatalf("merged slot: slots=%d names=%s", c.Slots, names) + } +} + +func TestPrivilegeRetriesFailedStop(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + if _, err := b.Replace(ctx, privBinding(1), privPlan("r1", privPolicy("p1", api.ActionAllow))); err != nil { + t.Fatal(err) + } + e.instances[0].stopErr = errors.New("busy") + r, err := b.Replace(ctx, privBinding(1), privPlan("r2", privPolicy("p1", api.ActionDeny))) + if err == nil || !r.Committed || r.ActiveRevision != "r2" { + t.Fatalf("committed with cleanup error expected: %+v %v", r, err) + } + e.instances[0].stopErr = nil + if _, err = b.Replace(ctx, privBinding(1), privPlan("r2", privPolicy("p1", api.ActionDeny))); err != nil || e.instances[0].stops != 2 { + t.Fatalf("retry: %v stops=%d", err, e.instances[0].stops) + } +} + +// An instance that is not ready may still run its programs, so a withdraw +// there stays pending: it is retried, and a new instance never carries it. +func TestPrivilegeWithdrawPendingWhenInstanceNotReady(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + if _, err := b.Replace(ctx, privBinding(1), privPlan("r1", privPolicy("p1", api.ActionAllow))); err != nil { + t.Fatal(err) + } + e.instances[0].readyErr = errors.New("not ready") + b.lastHeal = time.Now() // no recovery replacement in this part of the test + if r, err := b.Withdraw(ctx, privBinding(1)); err == nil || r.Committed || !r.ProtectionUnconfirmed { + t.Fatalf("withdraw on an instance that is not ready must stay pending: %+v %v", r, err) + } + // The retry on a ready instance unbinds and commits. + e.instances[0].readyErr = nil + if r, err := b.Withdraw(ctx, privBinding(1)); err != nil || !r.Committed { + t.Fatalf("retry: %+v %v", r, err) + } + if _, bound := e.instances[0].bound[1]; bound { + t.Fatal("the retry must unbind") + } + // Pending again, then a new instance replaces the old one: the withdrawn + // container is not carried, and the next retry commits. + if _, err := b.Replace(ctx, privBinding(1), privPlan("r1", privPolicy("p1", api.ActionAllow))); err != nil { + t.Fatal(err) + } + e.instances[0].readyErr = errors.New("not ready") + b.lastHeal = time.Now() + if r, _ := b.Withdraw(ctx, privBinding(1)); r.Committed { + t.Fatal("must stay pending") + } + if r, err := b.Replace(ctx, privBinding(2), privPlan("r2", privPolicy("p1", api.ActionAllow))); err != nil || !r.Committed { + t.Fatalf("recovery: %+v %v", r, err) + } + if c := e.configs[len(e.configs)-1]; len(c.InitialBindings) > 1 { + t.Fatalf("the withdrawn container must not be carried: %+v", c.InitialBindings) + } + if r, err := b.Withdraw(ctx, privBinding(1)); err != nil || !r.Committed { + t.Fatalf("after the replacement: %+v %v", r, err) + } +} + +func TestPrivilegeOversizedPrepareDoesNotBlockContainers(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + var plans []sandboxpolicy.Plan + for i := 0; i < 65; i++ { // one more than the slots per node + plans = append(plans, privPlan("p", privPolicy(string(rune('A'+i)), api.ActionAllow))) + } + if err := b.Prepare(ctx, plans); err == nil { + t.Fatal("too many combinations must fail") + } + if r, err := b.Replace(ctx, privBinding(1), privPlan("r1", privPolicy("real", api.ActionDeny))); err != nil || !r.Committed { + t.Fatalf("a real container must still bind: %+v %v", r, err) + } +} + +func TestPrivilegeUnreadyCandidateThatFailsToStopIsRetried(t *testing.T) { + e := &readyFailEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + if r, err := b.Replace(ctx, privBinding(1), privPlan("r1", privPolicy("p1", api.ActionAllow))); err == nil || r.Committed { + t.Fatalf("an unready candidate must fail: %+v", r) + } + if len(b.retiring) != 1 { + t.Fatal("a candidate that failed to stop must be kept for retry") + } + e.instance.stopErr = nil + if err := b.Close(ctx); err != nil || e.instance.stops != 2 { + t.Fatalf("close must retry the stop: %v stops=%d", err, e.instance.stops) + } +} + +type readyFailEngine struct{ instance *fakePrivilegeInstance } + +func (e *readyFailEngine) Start(context.Context, string, privruntime.Config) (PrivilegeInstance, error) { + e.instance = &fakePrivilegeInstance{readyErr: errors.New("not ready"), stopErr: errors.New("busy"), bound: map[uint64]uint32{}} + return e.instance, nil +} + +func TestPrivilegeTrackingFaultUnconfirmsAndHeals(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + plan := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + if _, err := b.Replace(ctx, privBinding(1), plan); err != nil { + t.Fatal(err) + } + e.instances[0].faults = map[uint32]uint64{e.instances[0].bound[1]: 1} + if b.Attest(ctx, privBinding(1), "r1") == nil { + t.Fatal("a tracking fault must fail attestation") + } + // The same plan again replaces the instance, which arms the tasks again. + if r, err := b.Replace(ctx, privBinding(1), plan); err != nil || !r.Committed || len(e.instances) != 2 { + t.Fatalf("heal: %+v %v instances=%d", r, err, len(e.instances)) + } + if err := b.Attest(ctx, privBinding(1), "r1"); err != nil { + t.Fatalf("the new instance has no fault: %v", err) + } + // A fault that recurs at once does not reload the gadget again. + e.instances[1].faults = map[uint32]uint64{e.instances[1].bound[1]: 1} + if r, err := b.Replace(ctx, privBinding(1), plan); err == nil || r.Committed || !r.ProtectionUnconfirmed || len(e.instances) != 2 { + t.Fatalf("heal within healRetry: %+v %v instances=%d", r, err, len(e.instances)) + } + b.lastHeal = b.lastHeal.Add(-healRetry) + if r, err := b.Replace(ctx, privBinding(1), plan); err != nil || !r.Committed || len(e.instances) != 3 { + t.Fatalf("heal after healRetry: %+v %v instances=%d", r, err, len(e.instances)) + } +} + +func testPodBinding(uid string, cgroup uint64, plan sandboxpolicy.Plan) sandboxpolicy.PodBinding { + return sandboxpolicy.PodBinding{PodUID: types.UID(uid), Namespace: "ns", PodName: uid, CgroupID: cgroup, CgroupPath: "/kubepods/pod" + uid, Plan: plan} +} + +func TestPrivilegePodBindings(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + p1 := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + p2 := privPlan("r2", privPolicy("p2", api.ActionDeny)) + // No slot yet: the pod waits for Prepare. + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, p1)}); err != nil || len(e.instances) != 0 { + t.Fatalf("no slot: %v instances %d", err, len(e.instances)) + } + if err := b.Prepare(ctx, []sandboxpolicy.Plan{p1}); err != nil { + t.Fatal(err) + } + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, p1)}); err != nil { + t.Fatal(err) + } + slot1, ok := e.instances[0].bound[100] + if !ok { + t.Fatal("the pod cgroup must be bound once its slot exists") + } + // A container of the pod binds to the same slot; the pod binding stays. + if _, err := b.Replace(ctx, privBinding(1), p1); err != nil || e.instances[0].bound[1] != slot1 || len(e.instances) != 1 { + t.Fatalf("container bind: %v %+v", err, e.instances[0].bound) + } + // A new combination replaces the instance; the pod binding is carried. + if err := b.Prepare(ctx, []sandboxpolicy.Plan{p1, p2}); err != nil || len(e.instances) != 2 { + t.Fatalf("prepare p2: %v", err) + } + carried := false + for _, initial := range e.configs[1].InitialBindings { + if initial.Container.CgroupID == 100 { + carried = initial.SkipArming && initial.Subtree && initial.RuntimeHelpers && initial.Container.Runtime == podRuntime && initial.Container.ID == "a" + } + } + if !carried { + t.Fatalf("pod binding not carried without arming: %+v", e.configs[1].InitialBindings) + } + // A changed plan moves the pod to the new slot in place. + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, p2)}); err != nil || e.instances[1].bound[100] == e.instances[1].bound[1] { + t.Fatalf("plan change: %v %+v", err, e.instances[1].bound) + } + // A recreated pod cgroup is a new identity: the old one is removed. + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 101, p2)}); err != nil { + t.Fatal(err) + } + if _, old := e.instances[1].bound[100]; old { + t.Fatal("the old pod cgroup must be unbound") + } + if _, ok := e.instances[1].bound[101]; !ok { + t.Fatal("the new pod cgroup must be bound") + } + // A pod that is not given any more is unbound; containers stay. + if err := b.BindPods(ctx, nil); err != nil { + t.Fatal(err) + } + if _, ok := e.instances[1].bound[101]; ok { + t.Fatal("pod binding must be removed") + } + if _, ok := e.instances[1].bound[1]; !ok { + t.Fatal("the container binding must stay") + } +} + +func TestPrivilegePodKeepsLastGoodUntilNewSlotIsReady(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + p1 := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + if err := b.Prepare(ctx, []sandboxpolicy.Plan{p1}); err != nil { + t.Fatal(err) + } + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, p1)}); err != nil { + t.Fatal(err) + } + old := e.instances[0].bound[100] + // An edited policy without a slot yet: the last-good binding stays. + edited := privPlan("r2", privPolicy("p1", api.ActionDeny)) + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, edited)}); err != nil { + t.Fatal(err) + } + if slot, ok := e.instances[0].bound[100]; !ok || slot != old { + t.Fatal("an edit without a ready slot must keep the last-good pod binding") + } + // Incomplete input (an empty plan) keeps it too. + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, sandboxpolicy.Plan{})}); err != nil { + t.Fatal(err) + } + if _, ok := e.instances[0].bound[100]; !ok { + t.Fatal("incomplete input must keep the pod binding") + } + // The new slot is ready: the pod moves to it. + if err := b.Prepare(ctx, []sandboxpolicy.Plan{p1, edited}); err != nil { + t.Fatal(err) + } + last := e.instances[len(e.instances)-1] + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, edited)}); err != nil { + t.Fatal(err) + } + if b.pods["a"] == nil || b.pods["a"].combination != func() string { c, _ := Combination(edited); return c }() { + t.Fatal("the pod must move to the new slot once it is ready") + } + if _, ok := last.bound[100]; !ok { + t.Fatal("the pod must be bound in the running instance") + } + // An authoritative removal unbinds it. + if err := b.BindPods(ctx, nil); err != nil { + t.Fatal(err) + } + if _, ok := last.bound[100]; ok { + t.Fatal("a pod that is not given must be unbound") + } +} + +// A candidate that does not fit may drop only the speculative prepared slots, +// never the active pod bindings; without prepared slots the new container is +// refused and the running instance keeps protecting the pods. +func TestPrivilegeRetryNeverDropsPodBindings(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + podPlan := privPlan("r1", privPolicy("pod", api.ActionAllow, denyMount)) + if err := b.Prepare(ctx, []sandboxpolicy.Plan{podPlan}); err != nil { + t.Fatal(err) + } + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, podPlan)}); err != nil { + t.Fatal(err) + } + spare := privPlan("r2", privPolicy("spare", api.ActionDeny)) + if err := b.Prepare(ctx, []sandboxpolicy.Plan{podPlan, spare}); err != nil { + t.Fatal(err) + } + // Room for two slots only. + e.failIf = func(c privruntime.Config) bool { return c.Policy.Slots > 2 } + if r, err := b.Replace(ctx, privBinding(1), privPlan("r3", privPolicy("new", api.ActionDeny))); err != nil || !r.Committed { + t.Fatalf("the retry without prepared slots must fit: %+v %v", r, err) + } + carried := false + for _, initial := range e.configs[len(e.configs)-1].InitialBindings { + carried = carried || initial.Container.CgroupID == 100 + } + if !carried { + t.Fatal("the retry must keep the pod binding") + } + // No prepared slots left to drop: a container that needs a third slot is refused. + e.failIf = func(c privruntime.Config) bool { return c.Policy.Slots > 2 } + running := e.instances[len(e.instances)-1] + if r, err := b.Replace(ctx, privBinding(2), privPlan("r4", privPolicy("third", api.ActionDeny))); err == nil || r.Committed { + t.Fatalf("a candidate that does not fit must be refused: %+v %v", r, err) + } + if _, bound := running.bound[100]; running.stops != 0 || !bound || b.pods["a"] == nil { + t.Fatal("the running instance must keep protecting the pod") + } +} + +func TestPrivilegePrepareStartsInstanceWithoutPlans(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + // A privilege policy exists but no pod uses it yet: load the gadget now, + // so the first container does not wait for the slow first load. + if err := b.Prepare(ctx, nil); err != nil || len(e.instances) != 1 || e.configs[0].Policy.Slots != 0 { + t.Fatalf("first prepare: %v instances=%d", err, len(e.instances)) + } + if err := b.Prepare(ctx, nil); err != nil || len(e.instances) != 1 { + t.Fatalf("a running instance must be kept: %v instances=%d", err, len(e.instances)) + } + plan := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + if _, err := b.Replace(ctx, privBinding(1), plan); err != nil || len(e.instances) != 2 { + t.Fatalf("first container: %v instances=%d", err, len(e.instances)) + } +} + +func TestPrivilegeEmptyInstanceRetryWaits(t *testing.T) { + e := &fakePrivilegeEngine{startErr: errors.New("load failed")} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + if err := b.Prepare(ctx, nil); err == nil { + t.Fatal("the failed start must be reported") + } + e.startErr = nil + if err := b.Prepare(ctx, nil); err != nil || len(e.instances) != 0 { + t.Fatalf("an empty start must not be retried at once: %v instances=%d", err, len(e.instances)) + } + // A real plan is not delayed. + if err := b.Prepare(ctx, []sandboxpolicy.Plan{privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount))}); err != nil || len(e.instances) != 1 { + t.Fatalf("prepare with a plan: %v instances=%d", err, len(e.instances)) + } +} + +func TestPrivilegeBoundPodPlansFollowBindings(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + gen := func(g int64) sandboxpolicy.Plan { + p := privPolicy("p1", api.ActionAllow, denyMount) + p.Identity.Generation = g + return privPlan(fmt.Sprintf("r%d", g), p) + } + if plans, faulted := b.BoundPodPlans(); len(plans)+len(faulted) != 0 { + t.Fatalf("no instance: %+v", plans) + } + if err := b.Prepare(ctx, []sandboxpolicy.Plan{gen(1)}); err != nil { + t.Fatal(err) + } + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, gen(1))}); err != nil { + t.Fatal(err) + } + if plans, _ := b.BoundPodPlans(); len(plans) != 1 || plans[0].Policies[0].Identity.Generation != 1 { + t.Fatalf("bound: %+v", plans) + } + // An edit with the same privilege content keeps the binding and reports the new generation. + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, gen(2))}); err != nil || len(e.instances) != 1 { + t.Fatalf("same content: %v instances %d", err, len(e.instances)) + } + if plans, _ := b.BoundPodPlans(); len(plans) != 1 || plans[0].Policies[0].Identity.Generation != 2 { + t.Fatalf("generation not refreshed: %+v", plans) + } + if err := b.BindPods(ctx, nil); err != nil { + t.Fatal(err) + } + if plans, faulted := b.BoundPodPlans(); len(plans)+len(faulted) != 0 { + t.Fatalf("unbound pod still reported: %+v", plans) + } +} + +// A pod plan counts only where the witness counters prove it: the pod was +// bound in the current instance before its containers started, and its slot +// has no fault. +func TestPrivilegePodPlansNeedWitness(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + p1 := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + p2 := privPlan("r2", privPolicy("p2", api.ActionDeny)) + if err := b.Prepare(ctx, []sandboxpolicy.Plan{p1}); err != nil { + t.Fatal(err) + } + started := testPodBinding("s", 200, p1) + started.ContainersStarted = true + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, p1), started}); err != nil { + t.Fatal(err) + } + if _, ok := e.instances[0].bound[200]; ok { + t.Fatal("a pod whose containers started must not get a pod binding") + } + if plans, faulted := b.BoundPodPlans(); len(plans) != 1 || len(faulted) != 0 { + t.Fatalf("bound before start: %d confirmed, %d faulted", len(plans), len(faulted)) + } + // A slot fault: the pod plan is faulted, not confirmed. + e.instances[0].faults = map[uint32]uint64{e.instances[0].bound[100]: 1} + if plans, faulted := b.BoundPodPlans(); len(plans) != 0 || len(faulted) != 1 { + t.Fatalf("slot fault: %d confirmed, %d faulted", len(plans), len(faulted)) + } + e.instances[0].faults = nil + // The pod's containers start, then a new instance carries the pod + // binding. The new instance arms none of the pod's running tasks, so the + // pod plan is no longer confirmed. + running := testPodBinding("a", 100, p1) + running.ContainersStarted = true + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{running}); err != nil { + t.Fatal(err) + } + if plans, _ := b.BoundPodPlans(); len(plans) != 1 { + t.Fatal("a pod bound before its containers started stays confirmed in its instance") + } + if err := b.Prepare(ctx, []sandboxpolicy.Plan{p1, p2}); err != nil || len(e.instances) != 2 { + t.Fatalf("prepare p2: %v", err) + } + if _, ok := e.instances[1].bound[100]; !ok { + t.Fatal("the pod binding must be carried") + } + if plans, faulted := b.BoundPodPlans(); len(plans) != 0 || len(faulted) != 0 { + t.Fatalf("carried after start: %d confirmed, %d faulted", len(plans), len(faulted)) + } +} + +// PrepareReplace creates the slot and binds nothing; the Replace that follows +// is a live map write in the same instance. +func TestPrivilegePrepareReplaceMakesBindLive(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + p1 := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + if err := b.PrepareReplace(ctx, privBinding(1), p1); err != nil || len(e.instances) != 1 { + t.Fatalf("prepare: %v instances %d", err, len(e.instances)) + } + if _, bound := e.instances[0].bound[1]; bound { + t.Fatal("prepare must not bind") + } + if r, err := b.Replace(ctx, privBinding(1), p1); err != nil || !r.Committed || len(e.instances) != 1 { + t.Fatalf("replace: %+v %v instances %d", r, err, len(e.instances)) + } + if b.Stats().Prepared != 1 || b.Stats().Replaced != 0 { + t.Fatalf("the bind must be a live write: %+v", b.Stats()) + } + // A prepared combination needs no new instance. + if err := b.PrepareReplace(ctx, privBinding(2), p1); err != nil || len(e.instances) != 1 { + t.Fatalf("second prepare: %v instances %d", err, len(e.instances)) + } +} + +// While the instance is not ready, every pod plan is faulted. +func TestPrivilegePodPlansFaultedWhileNotReady(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + p1 := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + if err := b.Prepare(ctx, []sandboxpolicy.Plan{p1}); err != nil { + t.Fatal(err) + } + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, p1)}); err != nil { + t.Fatal(err) + } + e.instances[0].readyErr = errors.New("not ready") + if plans, faulted := b.BoundPodPlans(); len(plans) != 0 || len(faulted) != 1 { + t.Fatalf("not ready: %d confirmed, %d faulted", len(plans), len(faulted)) + } +} + +// A bound pod whose containers started moves to its new slot when its plan +// changes, so a restarted container is never judged by a superseded plan; the +// move is not confirmed. +func TestPrivilegeStartedPodFollowsPlanChange(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + p1 := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + p2 := privPlan("r2", privPolicy("p1", api.ActionDeny)) + if err := b.Prepare(ctx, []sandboxpolicy.Plan{p1, p2}); err != nil { + t.Fatal(err) + } + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{testPodBinding("a", 100, p1)}); err != nil { + t.Fatal(err) + } + before := e.instances[0].bound[100] + moved := testPodBinding("a", 100, p2) + moved.ContainersStarted = true + if err := b.BindPods(ctx, []sandboxpolicy.PodBinding{moved}); err != nil { + t.Fatal(err) + } + if e.instances[0].bound[100] == before { + t.Fatal("the pod binding must move to the new plan's slot") + } + if plans, _ := b.BoundPodPlans(); len(plans) != 0 { + t.Fatal("a move after the containers started is not confirmed") + } +} + +// A Prepare between PrepareReplace and Replace keeps the reserved slot, so the +// Replace stays a live map write. +func TestPrivilegeReservedSlotSurvivesPrepare(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + p1 := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + p2 := privPlan("r2", privPolicy("p2", api.ActionDeny)) + if err := b.PrepareReplace(ctx, privBinding(1), p1); err != nil { + t.Fatal(err) + } + // The pod loop prepares another combination: a new instance. + if err := b.Prepare(ctx, []sandboxpolicy.Plan{p2}); err != nil || len(e.instances) != 2 { + t.Fatalf("prepare: %v instances %d", err, len(e.instances)) + } + if r, err := b.Replace(ctx, privBinding(1), p1); err != nil || !r.Committed || len(e.instances) != 2 { + t.Fatalf("replace must be live: %+v %v instances %d", r, err, len(e.instances)) + } +} + +// After healRetry, a withdraw on an instance that is not ready replaces it; +// the new instance never binds the withdrawn container. +func TestPrivilegePendingWithdrawReplacesInstance(t *testing.T) { + e := &fakePrivilegeEngine{} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + if _, err := b.Replace(ctx, privBinding(1), privPlan("r1", privPolicy("p1", api.ActionAllow))); err != nil { + t.Fatal(err) + } + e.instances[0].readyErr = errors.New("not ready") + if r, err := b.Withdraw(ctx, privBinding(1)); err != nil || !r.Committed || len(e.instances) != 2 { + t.Fatalf("recovery: %+v %v instances %d", r, err, len(e.instances)) + } + if _, bound := e.instances[1].bound[1]; bound { + t.Fatal("the new instance must not bind the withdrawn container") + } +} + +// A PrepareReplace whose combination does not fit reserves nothing, so it +// never blocks another workload's replacement. +func TestPrivilegeFailedPrepareReservesNothing(t *testing.T) { + big := privPlan("big", privPolicy("big", api.ActionDeny)) + e := &fakePrivilegeEngine{failIf: func(c privruntime.Config) bool { return c.Policy.Slots > 1 }} + b, _ := NewPrivilege(e, nil, "n") + ctx := context.Background() + p1 := privPlan("r1", privPolicy("p1", api.ActionAllow, denyMount)) + if _, err := b.Replace(ctx, privBinding(1), p1); err != nil { + t.Fatal(err) + } + if err := b.PrepareReplace(ctx, privBinding(2), big); err == nil { + t.Fatal("a combination that does not fit must fail") + } + if b.reserved != nil { + t.Fatalf("nothing may be reserved: %v", b.reserved) + } +} diff --git a/pkg/sandboxpolicy/integration/bindings.go b/pkg/sandboxpolicy/integration/bindings.go index 735fcedea3..2f92613056 100644 --- a/pkg/sandboxpolicy/integration/bindings.go +++ b/pkg/sandboxpolicy/integration/bindings.go @@ -13,6 +13,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" ) @@ -61,6 +62,85 @@ func SnapshotSelectedBindingsFromContainers(ctx context.Context, client kubernet return snapshotBindings(ctx, client, containers, nodeName, selector) } +// ScheduledContainers returns API metadata (no runtime identity) for every +// declared regular, init and ephemeral container of the pods scheduled to this +// node that are not finished, and the QoS class of each such pod (see +// scheduledContainers). It feeds privilege slot +// preparation and pod cgroup binding, not container binding. +func ScheduledContainers(ctx context.Context, client kubernetes.Interface, nodeName string) ([]sandboxpolicy.Binding, map[types.UID]corev1.PodQOSClass, map[types.UID]bool, error) { + if client == nil || nodeName == "" { + return nil, nil, nil, fmt.Errorf("Kubernetes client and node name are required") + } + pods, err := client.CoreV1().Pods("").List(ctx, metav1.ListOptions{FieldSelector: fields.OneTermEqualSelector("spec.nodeName", nodeName).String()}) + if err != nil { + return nil, nil, nil, fmt.Errorf("list node pods: %w", err) + } + namespaces, err := client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, nil, nil, fmt.Errorf("list namespaces: %w", err) + } + namespaceLabels := make(map[string]map[string]string, len(namespaces.Items)) + for _, ns := range namespaces.Items { + namespaceLabels[ns.Name] = ns.Labels + } + list := make([]*corev1.Pod, len(pods.Items)) + for i := range pods.Items { + list[i] = &pods.Items[i] + } + result, qos, started := scheduledContainers(list, namespaceLabels, nodeName) + return result, qos, started, nil +} + +// scheduledContainers is ScheduledContainers for a given pod list. A pod of a +// namespace without labels is skipped. A mirror (static) pod has no QoS class +// in the result: its pod cgroup is named after the kubelet's own UID, not the +// API UID, so it gets no pod binding. +func scheduledContainers(pods []*corev1.Pod, namespaceLabels map[string]map[string]string, nodeName string) ([]sandboxpolicy.Binding, map[types.UID]corev1.PodQOSClass, map[types.UID]bool) { + var result []sandboxpolicy.Binding + qos := map[types.UID]corev1.PodQOSClass{} + started := map[types.UID]bool{} + for _, pod := range pods { + if pod == nil || pod.Spec.NodeName != nodeName || pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { + continue + } + labels, ok := namespaceLabels[pod.Namespace] + if !ok { + continue + } + if _, mirror := pod.Annotations[corev1.MirrorPodAnnotationKey]; !mirror { + qos[pod.UID] = pod.Status.QOSClass + } + if podStarted(pod) { + started[pod.UID] = true + } + for _, group := range []struct { + kind string + containers []corev1.Container + }{{"regular", pod.Spec.Containers}, {"init", pod.Spec.InitContainers}} { + for _, c := range group.containers { + result = append(result, sandboxpolicy.Binding{PodUID: pod.UID, Namespace: pod.Namespace, PodName: pod.Name, ContainerName: c.Name, Kind: group.kind, PodLabels: copyLabels(pod.Labels), NamespaceLabels: copyLabels(labels)}) + } + } + for _, c := range pod.Spec.EphemeralContainers { + result = append(result, sandboxpolicy.Binding{PodUID: pod.UID, Namespace: pod.Namespace, PodName: pod.Name, ContainerName: c.Name, Kind: "ephemeral", PodLabels: copyLabels(pod.Labels), NamespaceLabels: copyLabels(labels)}) + } + } + return result, qos, started +} + +// podStarted reports whether the API shows a container of the pod as created: +// a status with a container ID, a running or terminated state, or a restart. +func podStarted(pod *corev1.Pod) bool { + for _, statuses := range [][]corev1.ContainerStatus{pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses, pod.Status.EphemeralContainerStatuses} { + for _, st := range statuses { + if st.ContainerID != "" || st.State.Running != nil || st.State.Terminated != nil || st.RestartCount > 0 || (st.Started != nil && *st.Started) { + return true + } + } + } + return false +} + func snapshotBindings(ctx context.Context, client kubernetes.Interface, containers []*containercollection.Container, nodeName string, selector func(sandboxpolicy.Binding) bool) ([]sandboxpolicy.Binding, error) { if client == nil || nodeName == "" { diff --git a/pkg/sandboxpolicy/integration/bindings_test.go b/pkg/sandboxpolicy/integration/bindings_test.go index 203472deeb..00279e1b43 100644 --- a/pkg/sandboxpolicy/integration/bindings_test.go +++ b/pkg/sandboxpolicy/integration/bindings_test.go @@ -257,3 +257,39 @@ func TestSnapshotRejectsInvalidMembershipDespiteMountedPath(t *testing.T) { }) } } + +func TestPodStarted(t *testing.T) { + started := true + for name, tc := range map[string]struct { + status corev1.PodStatus + want bool + }{ + "no status": {corev1.PodStatus{}, false}, + "waiting": {corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{}}}}}, false}, + "container ID": {corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{ContainerID: "containerd://abc"}}}, true}, + "running": {corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}}}, true}, + "init done": {corev1.PodStatus{InitContainerStatuses: []corev1.ContainerStatus{{State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{}}}}}, true}, + "started": {corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Started: &started}}}, true}, + } { + if got := podStarted(&corev1.Pod{Status: tc.status}); got != tc.want { + t.Errorf("%s: %v, want %v", name, got, tc.want) + } + } +} + +func TestUnderPod(t *testing.T) { + pod := "/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podabc.slice" + containers := []string{"/system.slice/docker.service", pod} + if underPod(containers, pod) { + t.Fatal("the pod cgroup itself is not a container under it") + } + if !underPod(append(containers, pod+"/cri-containerd-1.scope"), pod) { + t.Fatal("a container cgroup under the pod cgroup") + } + if !underPod([]string{"/host" + pod + "/cri-containerd-1.scope"}, pod) { + t.Fatal("a longer prefix still matches") + } + if underPod([]string{pod + "x/cri-containerd-1.scope"}, pod) { + t.Fatal("a sibling with the same prefix does not match") + } +} diff --git a/pkg/sandboxpolicy/integration/podcgroup.go b/pkg/sandboxpolicy/integration/podcgroup.go new file mode 100644 index 0000000000..ccb07d17d4 --- /dev/null +++ b/pkg/sandboxpolicy/integration/podcgroup.go @@ -0,0 +1,83 @@ +package integration + +import ( + "path" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +// podCgroupLayout is how the kubelet of this node names pod cgroups: the +// kubepods cgroup and the cgroup driver. The kubelet creates the pod cgroup +// before the pod sandbox, so a pod cgroup bound at that time covers the first +// exec of every container of the pod. +// +// systemd: /kubepods-burstable.slice/kubepods-burstable-pod.slice +// cgroupfs: /burstable/pod +// +// Guaranteed pods are directly under the root. The root (for example +// /kubepods.slice, or /kubelet.slice/kubelet-kubepods.slice in kind) is read +// from the cgroup path of a running pod container. +type podCgroupLayout struct { + root string + systemd bool +} + +// learnPodCgroupLayout reads the layout from the cgroup v2 path of a container +// of pod uid. It returns false when the path has no pod cgroup of that pod. +func learnPodCgroupLayout(cgroupPath string, uid types.UID) (podCgroupLayout, bool) { + if uid == "" || !path.IsAbs(cgroupPath) || path.Clean(cgroupPath) != cgroupPath { + return podCgroupLayout{}, false + } + parts := strings.Split(cgroupPath, "/") + for i := 2; i < len(parts); i++ { + for _, systemd := range []bool{true, false} { + // QoS levels first: a BestEffort pod cgroup also reads as a + // Guaranteed one under a root named after the QoS level. + for _, qos := range []corev1.PodQOSClass{corev1.PodQOSBestEffort, corev1.PodQOSBurstable, corev1.PodQOSGuaranteed} { + levels := 1 + if qos != corev1.PodQOSGuaranteed { + levels = 2 + } + if i-levels < 1 { + continue + } + layout := podCgroupLayout{root: strings.Join(parts[:i+1-levels], "/"), systemd: systemd} + if layout.path(uid, qos) == strings.Join(parts[:i+1], "/") { + return layout, true + } + } + } + } + return podCgroupLayout{}, false +} + +// path returns the pod cgroup path of a pod, or "" when the QoS class is not known. +func (l podCgroupLayout) path(uid types.UID, qos corev1.PodQOSClass) string { + if l.root == "" || uid == "" || strings.ContainsAny(string(uid), "/ \t\r\n") { + return "" + } + var level string + switch qos { + case corev1.PodQOSGuaranteed: + case corev1.PodQOSBurstable, corev1.PodQOSBestEffort: + level = strings.ToLower(string(qos)) + default: + return "" + } + if !l.systemd { + return path.Join(l.root, level, "pod"+string(uid)) + } + // systemd escapes "-" as "_" in a slice name; "-" separates the levels. + parent := strings.TrimSuffix(path.Base(l.root), ".slice") + if parent == path.Base(l.root) { + return "" + } + dir := l.root + if level != "" { + parent += "-" + level + dir = path.Join(dir, parent+".slice") + } + return path.Join(dir, parent+"-pod"+strings.ReplaceAll(string(uid), "-", "_")+".slice") +} diff --git a/pkg/sandboxpolicy/integration/podcgroup_linux.go b/pkg/sandboxpolicy/integration/podcgroup_linux.go new file mode 100644 index 0000000000..569b7492c8 --- /dev/null +++ b/pkg/sandboxpolicy/integration/podcgroup_linux.go @@ -0,0 +1,36 @@ +package integration + +import ( + "fmt" + "os" + "strings" + + "golang.org/x/sys/unix" +) + +// cgroupID returns the cgroup ID (the inode) of the cgroup v2 directory rel +// under root. Symlinks and paths outside root are refused. +func cgroupID(root, rel string) (uint64, error) { + dir, err := os.Open(root) + if err != nil { + return 0, err + } + defer dir.Close() + fd, err := unix.Openat2(int(dir.Fd()), strings.TrimPrefix(rel, "/"), &unix.OpenHow{Flags: unix.O_RDONLY | unix.O_DIRECTORY | unix.O_CLOEXEC, Resolve: unix.RESOLVE_BENEATH | unix.RESOLVE_NO_SYMLINKS}) + if err != nil { + return 0, err + } + defer unix.Close(fd) + var fs unix.Statfs_t + if err := unix.Fstatfs(fd, &fs); err != nil { + return 0, err + } + if fs.Type != unix.CGROUP2_SUPER_MAGIC { + return 0, fmt.Errorf("%s is not on cgroup v2", rel) + } + var stat unix.Stat_t + if err := unix.Fstat(fd, &stat); err != nil { + return 0, err + } + return stat.Ino, nil +} diff --git a/pkg/sandboxpolicy/integration/podcgroup_test.go b/pkg/sandboxpolicy/integration/podcgroup_test.go new file mode 100644 index 0000000000..1fd31e39cc --- /dev/null +++ b/pkg/sandboxpolicy/integration/podcgroup_test.go @@ -0,0 +1,72 @@ +package integration + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +const testPodUID = types.UID("0f3c2a1b-1111-2222-3333-444455556666") + +func TestPodCgroupLayout(t *testing.T) { + u := "0f3c2a1b_1111_2222_3333_444455556666" + other := types.UID("9e8d7c6b-aaaa-bbbb-cccc-ddddeeeeffff") + o := "9e8d7c6b_aaaa_bbbb_cccc_ddddeeeeffff" + for _, tc := range []struct { + name, container string + want map[corev1.PodQOSClass]string + }{ + {"systemd besteffort", "/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod" + u + ".slice/cri-containerd-abc.scope", map[corev1.PodQOSClass]string{ + corev1.PodQOSBestEffort: "/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod" + o + ".slice", + corev1.PodQOSBurstable: "/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod" + o + ".slice", + corev1.PodQOSGuaranteed: "/kubepods.slice/kubepods-pod" + o + ".slice", + }}, + {"systemd guaranteed, kind root", "/kubelet.slice/kubelet-kubepods.slice/kubelet-kubepods-pod" + u + ".slice/cri-containerd-abc.scope", map[corev1.PodQOSClass]string{ + corev1.PodQOSBurstable: "/kubelet.slice/kubelet-kubepods.slice/kubelet-kubepods-burstable.slice/kubelet-kubepods-burstable-pod" + o + ".slice", + corev1.PodQOSGuaranteed: "/kubelet.slice/kubelet-kubepods.slice/kubelet-kubepods-pod" + o + ".slice", + }}, + {"cgroupfs burstable", "/kubepods/burstable/pod" + string(testPodUID) + "/abc", map[corev1.PodQOSClass]string{ + corev1.PodQOSBestEffort: "/kubepods/besteffort/pod" + string(other), + corev1.PodQOSGuaranteed: "/kubepods/pod" + string(other), + }}, + {"cgroupfs guaranteed", "/kubepods/pod" + string(testPodUID) + "/abc", map[corev1.PodQOSClass]string{ + corev1.PodQOSBurstable: "/kubepods/burstable/pod" + string(other), + }}, + } { + t.Run(tc.name, func(t *testing.T) { + layout, ok := learnPodCgroupLayout(tc.container, testPodUID) + if !ok { + t.Fatalf("layout not learned from %s", tc.container) + } + for qos, want := range tc.want { + if got := layout.path(other, qos); got != want { + t.Errorf("%s: got %s, want %s", qos, got, want) + } + } + if got := layout.path(other, ""); got != "" { + t.Errorf("unknown QoS class must give no path: %s", got) + } + }) + } +} + +func TestPodCgroupLayoutRefusesOtherPaths(t *testing.T) { + for _, path := range []string{ + "/system.slice/docker-abc.scope", + "/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podother.slice/cri-containerd-abc.scope", + "kubepods/pod" + string(testPodUID), + "/kubepods/../pod" + string(testPodUID), + "/pod" + string(testPodUID), + } { + if layout, ok := learnPodCgroupLayout(path, testPodUID); ok { + t.Errorf("%s: unexpected layout %+v", path, layout) + } + } + if _, ok := learnPodCgroupLayout("/kubepods/pod"+string(testPodUID), ""); ok { + t.Error("a container without a pod UID must not give a layout") + } + if got := (podCgroupLayout{root: "/kubepods"}).path("a/b", corev1.PodQOSGuaranteed); got != "" { + t.Errorf("a UID with a separator must give no path: %s", got) + } +} diff --git a/pkg/sandboxpolicy/integration/service.go b/pkg/sandboxpolicy/integration/service.go index a6ca6ddffc..e8284674b6 100644 --- a/pkg/sandboxpolicy/integration/service.go +++ b/pkg/sandboxpolicy/integration/service.go @@ -4,7 +4,9 @@ import ( "context" "errors" "fmt" + stdmaps "maps" "path/filepath" + "strings" "sync" "sync/atomic" "time" @@ -23,10 +25,16 @@ import ( "github.com/kubescape/node-agent/pkg/utils" "github.com/kubescape/node-agent/pkg/watcher" api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" + "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" orasoci "oras.land/oras-go/v2/content/oci" ) @@ -44,6 +52,13 @@ type ServiceOptions struct { Receivers *maps.SafeMap[utils.EventType, mapset.Set[containerwatcher.GenericEventReceiver]] Enricher containerwatcher.TaskBasedEnricher OnError func(error) + // PrivilegeImageName enables the privilege section. Empty rejects it. + PrivilegeImageName string + // QualifiedKernels lists the kernel releases (uname -r) on which the + // privilege section may report Ready; empty uses igbackend.QualifiedKernels, + // and "*" admits every kernel. On another kernel it enforces but stays + // NotReady (PrivilegeKernelUnqualified). + QualifiedKernels []string } type serviceBackend interface { @@ -70,8 +85,43 @@ type Service struct { wake chan struct{} inputEpoch atomic.Uint64 snapshotEpoch atomic.Uint64 + // podsScheduled is set by a Pending-pod or policy event; lastPrepare + // bounds how often privilege slot preparation lists the node's pods. + podsScheduled atomic.Bool + lastPrepare time.Time + // The fields below are used only by the refresh loop. + // scheduled and qos are the containers and QoS classes of the node's pods + // from the last slot preparation, used when there are no informers. + scheduled []sandboxpolicy.Binding + qos map[types.UID]corev1.PodQOSClass + started map[types.UID]bool + // podFirstSeen is when a pod with a pod plan was first seen. podPolling is + // set while such a pod has no pod cgroup yet, for up to prepareInterval. + podFirstSeen map[types.UID]time.Time + podPolling bool + // podPods, podQoS and podPlans are the inputs of the last bindPods, + // reused by the poll. podPollErr is the last error the poll reported. + podPods []sandboxpolicy.Binding + podQoS map[types.UID]corev1.PodQOSClass + podStarted map[types.UID]bool + podPlans map[types.UID]sandboxpolicy.Plan + podPollErr string + // podCgroupID returns the cgroup ID of a pod cgroup path (replaced in tests). + podCgroupID func(string) (uint64, error) + // pods and namespaces are the informers of the node's pods and of all + // namespaces (nil without a privilege image). + pods, namespaces informers.SharedInformerFactory } +// prepareInterval is the longest time between two slot preparations. +const prepareInterval = 30 * time.Second + +// podPollInterval is how often a scheduled pod is checked for its pod cgroup, +// for up to prepareInterval after it is scheduled. The kubelet creates the pod +// cgroup before the pod sandbox, so the pod cgroup is bound before the first +// container starts. +const podPollInterval = 50 * time.Millisecond + // NewService prepares the public watcher/exporter/native runtime bridge. It does // not authorize any cgroup or start enforcement until Start is called. func NewService(ctx context.Context, options ServiceOptions) (*Service, error) { @@ -91,6 +141,15 @@ func NewService(ctx context.Context, options ServiceOptions) (*Service, error) { return nil, fmt.Errorf("sandbox native runtime: %w", err) } s := &Service{options: options, bindings: map[string]sandboxpolicy.Binding{}, closeRuntime: r.Close, done: make(chan struct{}), wake: make(chan struct{}, 1)} + cgroupRoot := filepath.Join(host.HostRoot, "sys/fs/cgroup") + s.podCgroupID = func(path string) (uint64, error) { return cgroupID(cgroupRoot, path) } + if options.PrivilegeImageName != "" { + s.pods = informers.NewSharedInformerFactoryWithOptions(options.Kubernetes, 0, informers.WithTweakListOptions(func(o *metav1.ListOptions) { + o.FieldSelector = fields.OneTermEqualSelector("spec.nodeName", options.NodeName).String() + })) + s.namespaces = informers.NewSharedInformerFactory(options.Kubernetes, 0) + s.watchPods() + } s.exporter, err = sandboxpolicy.NewVerdictExporter(sandboxpolicy.ExporterOptions{Capacity: 1024, Receivers: options.Receivers, Enricher: options.Enricher, ResolveBinding: s.resolveBinding}) if err != nil { _ = r.Close() @@ -119,12 +178,44 @@ func NewService(ctx context.Context, options ServiceOptions) (*Service, error) { _ = s.closeRuntime() return nil, err } - s.backend, err = igbackend.NewFull(engine, source, s.exporter, s.report) + full, err := igbackend.NewFull(engine, source, s.exporter, s.report) + if err != nil { + _ = s.closeRuntime() + return nil, err + } + var privilege sandboxpolicy.Backend + var privilegeUnavailable error + lsmUnavailable := igbackend.CheckBPFLSM(host.HostRoot) + if options.PrivilegeImageName != "" { + privilegeUnavailable = lsmUnavailable + } + if options.PrivilegeImageName != "" && privilegeUnavailable == nil { + privilegeEngine, err := igbackend.NewNativePrivilegeEngine(r, kubemanager.NewKubeManager(options.Containers, options.Tracers), store, options.PrivilegeImageName) + if err == nil { + privilege, err = igbackend.NewPrivilege(privilegeEngine, igbackend.LinuxValidator{ProcRoot: host.HostProcFs, CgroupRoot: filepath.Join(host.HostRoot, "sys/fs/cgroup")}, options.NodeName) + } + if err != nil { + _ = full.Close(ctx) + _ = s.closeRuntime() + return nil, err + } + } + sections, err := sandboxpolicy.NewSectionBackend(full, privilege, privilegeUnavailable) if err != nil { + _ = full.Close(ctx) _ = s.closeRuntime() return nil, err } - s.Controller, err = sandboxpolicy.NewController(options.Dynamic, s.backend, options.Sources, readinessWriter{inner: &sandboxpolicy.DynamicStatusWriter{Client: options.Dynamic}, observed: &s.inputEpoch, snapshot: &s.snapshotEpoch}, sandboxpolicy.Options{Node: api.NodeReference{Name: node.Name, UID: node.UID}, AgentNamespace: options.AgentNamespace}) + if privilege != nil { + release, rerr := igbackend.KernelRelease() + if rerr != nil { + sections.SetPrivilegeUnqualified(&sandboxpolicy.EnforcementUnavailableError{Section: "privilege", Reason: "PrivilegeKernelUnqualified", Enforcing: true, Detail: fmt.Sprintf("kernel release: %v", rerr)}) + } else { + sections.SetPrivilegeUnqualified(igbackend.CheckQualifiedKernel(release, options.QualifiedKernels)) + } + } + s.backend = sections + s.Controller, err = sandboxpolicy.NewController(options.Dynamic, s.backend, options.Sources, readinessWriter{inner: &sandboxpolicy.DynamicStatusWriter{Client: options.Dynamic}, observed: &s.inputEpoch, snapshot: &s.snapshotEpoch}, sandboxpolicy.Options{Node: api.NodeReference{Name: node.Name, UID: node.UID}, AgentNamespace: options.AgentNamespace, EnforcementUnavailable: lsmUnavailable}) if err != nil { _ = s.backend.Close(ctx) _ = s.closeRuntime() @@ -136,7 +227,9 @@ func NewService(ctx context.Context, options ServiceOptions) (*Service, error) { // Watch events are hints only. Generic watcher queues may deliver old updates // after a relist/deletion; applying their payload could resurrect removed policy. // Refresh always reads authoritative configuration and cannot revive queued UIDs. -func (s *Service) WatchResources() []watcher.WatchResource { return s.Controller.WatchResources() } +func (s *Service) WatchResources() []watcher.WatchResource { + return s.Controller.WatchResources() +} func (s *Service) AddHandler(_ context.Context, obj runtime.Object) { s.configurationHint(obj) } func (s *Service) ModifyHandler(_ context.Context, obj runtime.Object) { s.configurationHint(obj) } func (s *Service) DeleteHandler(_ context.Context, obj runtime.Object) { s.configurationHint(obj) } @@ -146,10 +239,56 @@ func (s *Service) configurationHint(obj runtime.Object) { } switch obj.GetObjectKind().GroupVersionKind().Kind { case "SandboxPolicy", "ClusterSandboxPolicy", "SandboxPolicyClasses": + // A policy change can need a new slot for the pod bindings. + s.podsScheduled.Store(true) s.ContainerCallback(containercollection.PubSubEvent{}) } } +// watchPods wakes the refresh loop on changes of the node's pods. A Pending pod +// also asks for a slot preparation. A label or ephemeral container change can +// change the pod plan, so the pod bindings are computed again. It is a +// dedicated informer: the generic watcher delivers an event only after the +// object has had no update for 5 s, which is later than the start of the +// pod's containers. +func (s *Service) watchPods() { + hint := func(old, obj any) { + pod, ok := obj.(*corev1.Pod) + if !ok { + s.ContainerCallback(containercollection.PubSubEvent{}) // deleted + return + } + if pod.Status.Phase == corev1.PodPending { + s.podsScheduled.Store(true) + s.ContainerCallback(containercollection.PubSubEvent{}) + return + } + if before, ok := old.(*corev1.Pod); !ok || !stdmaps.Equal(before.Labels, pod.Labels) || len(before.Spec.EphemeralContainers) != len(pod.Spec.EphemeralContainers) { + s.ContainerCallback(containercollection.PubSubEvent{}) + } + } + _, _ = s.namespaces.Core().V1().Namespaces().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + // A pod can arrive before its new namespace: it was dropped, so + // prepare again. + AddFunc: func(any) { + s.podsScheduled.Store(true) + s.ContainerCallback(containercollection.PubSubEvent{}) + }, + UpdateFunc: func(old, obj any) { + before, ok1 := old.(*corev1.Namespace) + after, ok2 := obj.(*corev1.Namespace) + if !ok1 || !ok2 || !stdmaps.Equal(before.Labels, after.Labels) { + s.ContainerCallback(containercollection.PubSubEvent{}) + } + }, + }) + _, _ = s.pods.Core().V1().Pods().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { hint(nil, obj) }, + UpdateFunc: hint, + DeleteFunc: func(any) { hint(nil, nil) }, + }) +} + // Start is called after container discovery is running and before the generic // dynamic watcher starts. Relisting repairs watch disconnections and re-resolves // source profiles/selector labels. Failed snapshots never become an empty list. @@ -174,6 +313,15 @@ func (s *Service) Start(ctx context.Context) error { s.report(fmt.Errorf("sandbox verdict exporter: %w", err)) } }() + if s.pods != nil { + s.pods.Start(runCtx.Done()) + s.namespaces.Start(runCtx.Done()) + // Bounded: the caches only add hints, refresh works without them. + syncCtx, cancelSync := context.WithTimeout(runCtx, 10*time.Second) + s.pods.WaitForCacheSync(syncCtx.Done()) + s.namespaces.WaitForCacheSync(syncCtx.Done()) + cancelSync() + } // Drain observed startup churn into the readiness barrier before handing // work to the background loop. Persistent churn stays explicitly NotReady. drain: @@ -197,6 +345,10 @@ drain: ticker := time.NewTicker(5 * time.Second) defer ticker.Stop() for { + var poll <-chan time.Time + if s.podPolling { + poll = time.After(podPollInterval) + } select { case <-runCtx.Done(): return @@ -204,6 +356,8 @@ drain: s.refresh(runCtx) case <-s.wake: s.refresh(runCtx) + case <-poll: + s.pollPods(runCtx) } } }() @@ -211,6 +365,24 @@ drain: } func (s *Service) refresh(ctx context.Context) { s.snapshotEpoch.Store(s.inputEpoch.Load()) + // Prepare after a Pending-pod or policy event, else at most every + // prepareInterval. + if s.options.PrivilegeImageName != "" && s.options.Kubernetes != nil && s.Controller.HasPrivilegePolicies() && + (s.podsScheduled.Swap(false) || time.Since(s.lastPrepare) >= prepareInterval) { + s.lastPrepare = time.Now() + scheduled, qos, started, err := s.scheduledPods(ctx) + if err == nil { + s.scheduled, s.qos, s.started = scheduled, qos, started + err = s.Controller.PrepareScheduled(ctx, scheduled) + } + if err != nil { + s.report(fmt.Errorf("privilege slot preparation: %w", err)) + } + } + // Every refresh, so a policy change also reaches the pod bindings: before + // the container bindings change, and after, when they created a slot. + s.bindPods(ctx) + defer s.bindPods(ctx) // Configuration and selected-runtime discovery share one commit barrier. A // candidate cannot report Ready between its relist and a failed binding list. err := s.Controller.Refresh(ctx, func(ctx context.Context, selected func(sandboxpolicy.Binding) bool) ([]sandboxpolicy.Binding, error) { @@ -242,6 +414,170 @@ func (s *Service) refresh(ctx context.Context) { } } +// bindPods binds the pod cgroup of every scheduled pod that has a pod plan +// (see Controller.PodPlans) and unbinds the others. It reads the node's pods +// from the informer cache and makes no API call. A pod whose cgroup does not +// exist yet sets podPolling, so it is checked again soon, for up to +// prepareInterval after it was first seen. +func (s *Service) bindPods(ctx context.Context) { + if _, ok := s.backend.(sandboxpolicy.PodBinder); !ok || s.Controller == nil { + return + } + s.podPolling = false // set again below when a pod cgroup is missing + scheduled, qos, started := s.scheduled, s.qos, s.started + if s.pods != nil { + var err error + if scheduled, qos, started, err = s.scheduledPods(ctx); err != nil { + s.report(fmt.Errorf("privilege pod binding: %w", err)) + return + } + } + plans, ok := s.Controller.PodPlans(scheduled) + if !ok { + s.podPlans = nil + return // the policies are not known: keep the pod bindings + } + s.podPods, s.podQoS, s.podStarted, s.podPlans = scheduled, qos, started, plans + s.podPollErr = "" + s.bindPodPlans(ctx, scheduled, qos, started, plans, false) +} + +// pollPods checks again for the pod cgroups that were missing. It reuses the +// pods and plans of the last bindPods, so the 50 ms poll does not list pods +// and compile policies. +func (s *Service) pollPods(ctx context.Context) { + s.podPolling = false + if s.podPlans == nil { + return + } + s.bindPodPlans(ctx, s.podPods, s.podQoS, s.podStarted, s.podPlans, true) +} + +func (s *Service) bindPodPlans(ctx context.Context, scheduled []sandboxpolicy.Binding, qos map[types.UID]corev1.PodQOSClass, apiStarted map[types.UID]bool, known map[types.UID]sandboxpolicy.Plan, poll bool) { + binder := s.backend.(sandboxpolicy.PodBinder) + plans := make(map[types.UID]sandboxpolicy.Plan, len(known)) + for k, v := range known { + plans[k] = v + } + var pods []sandboxpolicy.PodBinding + now := time.Now() + firstSeen := map[types.UID]time.Time{} + polling := false + if len(plans) > 0 { + layout, known, containerCgroups, started, containerPaths := s.podCgroupLayout() + for _, c := range scheduled { + plan, ok := plans[c.PodUID] + if !ok { + continue + } + delete(plans, c.PodUID) + seen, ok := s.podFirstSeen[c.PodUID] + if !ok { + seen = now + } + firstSeen[c.PodUID] = seen + path := layout.path(c.PodUID, qos[c.PodUID]) + if !known || path == "" || s.podCgroupID == nil { + continue + } + id, err := s.podCgroupID(path) + if err != nil || id == 0 { + polling = polling || now.Sub(seen) < prepareInterval + continue + } + // A runtime that runs the container in the pod cgroup itself: + // the container binding owns it. + if containerCgroups[id] { + continue + } + pods = append(pods, sandboxpolicy.PodBinding{PodUID: c.PodUID, Namespace: c.Namespace, PodName: c.PodName, CgroupID: id, CgroupPath: path, Plan: plan, + ContainersStarted: started[c.PodUID] || apiStarted[c.PodUID] || underPod(containerPaths, path)}) + } + } + s.podFirstSeen, s.podPolling = firstSeen, polling + if err := binder.BindPods(ctx, pods); err != nil { + // The poll runs every 50 ms: report a repeated error once. + if !poll || err.Error() != s.podPollErr { + s.report(fmt.Errorf("privilege pod binding: %w", err)) + } + if poll { + s.podPollErr = err.Error() + } + } +} + +// scheduledPods returns the containers and QoS classes of the node's pods (see +// ScheduledContainers), from the informer caches when there are informers. +func (s *Service) scheduledPods(ctx context.Context) ([]sandboxpolicy.Binding, map[types.UID]corev1.PodQOSClass, map[types.UID]bool, error) { + if s.pods == nil { + return ScheduledContainers(ctx, s.options.Kubernetes, s.options.NodeName) + } + pods, err := s.pods.Core().V1().Pods().Lister().List(labels.Everything()) + if err != nil { + return nil, nil, nil, err + } + namespaces, err := s.namespaces.Core().V1().Namespaces().Lister().List(labels.Everything()) + if err != nil { + return nil, nil, nil, err + } + namespaceLabels := make(map[string]map[string]string, len(namespaces)) + for _, ns := range namespaces { + namespaceLabels[ns.Name] = ns.Labels + } + scheduled, qos, started := scheduledContainers(pods, namespaceLabels, s.options.NodeName) + return scheduled, qos, started, nil +} + +// podCgroupLayout reads the pod cgroup layout from a running pod container, +// and returns the cgroup IDs of the running containers. +func (s *Service) podCgroupLayout() (podCgroupLayout, bool, map[uint64]bool, map[types.UID]bool, []string) { + var layout podCgroupLayout + found := false + cgroups := map[uint64]bool{} + // The pods with a container that exists, by pod UID and by cgroup path + // (a container whose pod UID is not enriched yet). The collection drops + // pause containers. Together with the pod's API status this is a + // snapshot: a container created after it is not seen until the next + // call (see sandboxpolicy.PodBinding). + started := map[types.UID]bool{} + var paths []string + if s.options.Containers == nil { + return layout, false, cgroups, started, paths + } + s.options.Containers.ContainerRange(func(c *containercollection.Container) { + if c == nil { + return + } + cgroups[c.CgroupID] = true + if c.K8s.PodUID != "" { + started[types.UID(c.K8s.PodUID)] = true + } + if c.CgroupV2 != "" { + paths = append(paths, c.CgroupV2) + } + if !found { + layout, found = learnPodCgroupLayout(c.CgroupV2, types.UID(c.K8s.PodUID)) + } + }) + return layout, found, cgroups, started, paths +} + +// underPod reports whether a container cgroup path lies under the pod cgroup +// path. Both come from the same host view; the container path may carry a +// longer prefix, so the pod path is matched as a path segment sequence. +func underPod(containerPaths []string, podPath string) bool { + pod := "/" + strings.Trim(podPath, "/") + "/" + if pod == "//" { + return false + } + for _, p := range containerPaths { + if strings.Contains(p+"/", pod) && !strings.HasSuffix(strings.TrimRight(p, "/")+"/", pod) { + return true + } + } + return false +} + // ContainerCallback is deliberately nonblocking: event delivery must never hold // discovery locks while a gadget is loading or Kubernetes is unavailable. func (s *Service) ContainerCallback(containercollection.PubSubEvent) { @@ -288,6 +624,10 @@ func (s *Service) Close(ctx context.Context) error { case <-ctx.Done(): return ctx.Err() } + if s.pods != nil { + s.pods.Shutdown() + s.namespaces.Shutdown() + } } if !s.backendClosed { if err := s.backend.Close(ctx); err != nil { @@ -308,7 +648,11 @@ func (s *Service) Close(ctx context.Context) error { // from ExporterStats. Unavailable counts/errors mean totals are incomplete; // these evaluator counters do not cover early native-owner exec refusals. func (s *Service) KernelEventStats(ctx context.Context) (igbackend.KernelEventStats, error) { - source, ok := s.backend.(interface { + var backend any = s.backend + if sections, isSections := s.backend.(*sandboxpolicy.SectionBackend); isSections { + backend = sections.Exec() + } + source, ok := backend.(interface { KernelEventStats(context.Context) (igbackend.KernelEventStats, error) }) if !ok { diff --git a/pkg/sandboxpolicy/integration/service_test.go b/pkg/sandboxpolicy/integration/service_test.go index 535c6925a3..c9be915a26 100644 --- a/pkg/sandboxpolicy/integration/service_test.go +++ b/pkg/sandboxpolicy/integration/service_test.go @@ -11,10 +11,12 @@ import ( containercollection "github.com/inspektor-gadget/inspektor-gadget/pkg/container-collection" "github.com/kubescape/node-agent/pkg/sandboxpolicy" api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" dynamicfake "k8s.io/client-go/dynamic/fake" ) @@ -267,6 +269,98 @@ func TestServiceDrainsStartupChurnBeforeReady(t *testing.T) { } } +type podBinderBackend struct { + serviceTestBackend + pods [][]sandboxpolicy.PodBinding + err error +} + +func (b *podBinderBackend) BindPods(_ context.Context, pods []sandboxpolicy.PodBinding) error { + b.pods = append(b.pods, pods) + return b.err +} + +func TestServiceBindsPodCgroups(t *testing.T) { + client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{sandboxpolicy.PolicyGVR: "SandboxPolicyList", sandboxpolicy.ClusterPolicyGVR: "ClusterSandboxPolicyList", sandboxpolicy.ClassesGVR: "SandboxPolicyClassesList"}) + p := &unstructured.Unstructured{Object: map[string]any{"apiVersion": api.SchemeGroupVersion.String(), "kind": "SandboxPolicy", "metadata": map[string]any{"name": "priv", "namespace": "ns", "uid": "policy-uid", "generation": int64(1)}, "spec": map[string]any{"selector": map[string]any{"podSelector": map[string]any{}}, "mode": "enforce", "containers": []any{"*"}, "privilege": map[string]any{"default": "allow", "rules": []any{map[string]any{"name": "no-chroot", "action": "deny", "target": map[string]any{"syscall": []any{"chroot"}}}}}}}} + if _, err := client.Resource(sandboxpolicy.PolicyGVR).Namespace("ns").Create(t.Context(), p, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + backend := &podBinderBackend{} + controller, err := sandboxpolicy.NewController(client, backend, nil, nil, sandboxpolicy.Options{Node: api.NodeReference{Name: "node", UID: "node-uid"}, AgentNamespace: "agent"}) + if err != nil { + t.Fatal(err) + } + if err := controller.InitialSync(t.Context()); err != nil { + t.Fatal(err) + } + // A running container of another pod gives the layout. + containers := &containercollection.ContainerCollection{} + running := &containercollection.Container{CgroupID: 500, CgroupV2: "/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podother_uid.slice/cri-containerd-x.scope"} + running.Runtime.ContainerID = "x" + running.K8s.PodUID = "other-uid" + containers.AddContainer(running) + want := "/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podpod_uid.slice" + var id uint64 + s := &Service{Controller: controller, backend: backend, options: ServiceOptions{Containers: containers, NodeName: "node"}, + scheduled: []sandboxpolicy.Binding{{PodUID: "pod-uid", Namespace: "ns", PodName: "pod", ContainerName: "app", Kind: "regular"}}, + qos: map[types.UID]corev1.PodQOSClass{"pod-uid": corev1.PodQOSBestEffort}, + podCgroupID: func(path string) (uint64, error) { + if path != want || id == 0 { + return 0, errors.New("no such cgroup") + } + return id, nil + }} + last := func() []sandboxpolicy.PodBinding { return backend.pods[len(backend.pods)-1] } + + // The pod cgroup does not exist yet: nothing is bound, and the pod is polled. + s.bindPods(t.Context()) + if len(last()) != 0 || !s.podPolling { + t.Fatalf("missing cgroup: %+v polling=%v", last(), s.podPolling) + } + // The poll reuses the last plans: it binds once the cgroup appears, + // and it reports a repeated BindPods error once. + id = 77 + s.pollPods(t.Context()) + if got := last(); len(got) != 1 || got[0].CgroupID != 77 || s.podPolling { + t.Fatalf("poll bind: %+v polling=%v", got, s.podPolling) + } + reports := 0 + s.options.OnError = func(error) { reports++ } + backend.err = errors.New("busy") + s.pollPods(t.Context()) + s.pollPods(t.Context()) + backend.err = nil + s.options.OnError = nil + if reports != 1 { + t.Fatalf("a repeated poll error must be reported once: %d", reports) + } + s.bindPods(t.Context()) + if got := last(); len(got) != 1 || got[0].CgroupID != 77 || got[0].CgroupPath != want || len(got[0].Plan.Policies) != 1 || s.podPolling { + t.Fatalf("bind: %+v polling=%v", got, s.podPolling) + } + // A container that runs in the pod cgroup itself keeps its own binding. + id = 500 + s.bindPods(t.Context()) + if len(last()) != 0 { + t.Fatalf("pod cgroup shared with a container must not be bound: %+v", last()) + } + // Polling stops when the pod was first seen too long ago. + id = 0 + s.podFirstSeen["pod-uid"] = time.Now().Add(-prepareInterval) + s.bindPods(t.Context()) + if s.podPolling { + t.Fatal("polling must stop after prepareInterval") + } + // An init container that the wildcard does not select: no pod binding. + id = 77 + s.scheduled = append(s.scheduled, sandboxpolicy.Binding{PodUID: "pod-uid", Namespace: "ns", PodName: "pod", ContainerName: "setup", Kind: "init"}) + s.bindPods(t.Context()) + if len(last()) != 0 { + t.Fatalf("mixed pod must not be bound: %+v", last()) + } +} + func TestServiceReportsUnexpectedExporterExit(t *testing.T) { s, _ := testService(t) stopped, cancel := context.WithCancel(context.Background()) diff --git a/pkg/sandboxpolicy/privilege.go b/pkg/sandboxpolicy/privilege.go new file mode 100644 index 0000000000..06b285210d --- /dev/null +++ b/pkg/sandboxpolicy/privilege.go @@ -0,0 +1,38 @@ +package sandboxpolicy + +import ( + "fmt" + + privpolicy "github.com/kubescape/node-agent/pkg/ebpf/gadgets/enforce_privilege/policy" + api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" +) + +// PrivilegeDocument translates the privilege section of one SandboxPolicy into +// the privilege gadget's own policy document. name qualifies rule names in +// events. Actor, pathClass and except are not supported by the gadget yet and +// are rejected. Origins keep their presence: an explicit "" is not a missing +// origin. +// +// Several policies on one container merge per the shared merge contract +// (pkg/ebpf/gadgets/common/merge/CONTRACT.md). It differs from the CRD design +// text (sandbox-policy-crd.md, "Multiple policies matching one pod merge"): an +// audit deny rule never weakens another policy's enforce default deny. +func PrivilegeDocument(name string, mode api.PolicyMode, section api.PrivilegeSection) (privpolicy.Document, error) { + doc := privpolicy.Document{Name: name, Version: 1, Policy: privpolicy.Policy{Mode: string(mode), Privilege: privpolicy.Section{ + Default: string(section.Default), DefaultMode: string(section.DefaultMode), DefaultOrigin: section.DefaultOrigin, + }}} + for _, r := range section.Rules { + t := r.Target + if r.Actor != nil || len(t.PathClass) > 0 { + return doc, fmt.Errorf("rule %q: actor and pathClass targets are unsupported in the privilege section", r.Name) + } + if len(r.Except) > 0 { + return doc, fmt.Errorf("rule %q: exceptions are unsupported in the privilege section", r.Name) + } + doc.Policy.Privilege.Rules = append(doc.Policy.Privilege.Rules, privpolicy.Rule{ + Name: r.Name, Action: string(r.Action), Mode: string(r.Mode), Origin: r.Origin, + Target: privpolicy.Target{Syscall: t.Syscall, Flags: t.Flags, Capability: t.Capability, MountFSType: t.MountFSType}, + }) + } + return doc, doc.Validate() +} diff --git a/pkg/sandboxpolicy/privilege_test.go b/pkg/sandboxpolicy/privilege_test.go new file mode 100644 index 0000000000..ae74e96018 --- /dev/null +++ b/pkg/sandboxpolicy/privilege_test.go @@ -0,0 +1,63 @@ +package sandboxpolicy + +import ( + "strings" + "testing" + + api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" +) + +func TestPrivilegeDocumentRejectsFlaglessClone(t *testing.T) { + section := func(syscall string, flags ...string) api.PrivilegeSection { + return api.PrivilegeSection{Default: api.ActionAllow, Rules: []api.PrivilegeRule{{ + Rule: api.Rule{Name: "r", Action: api.ActionDeny}, + Target: api.PrivilegeTarget{Syscall: []string{syscall}, Flags: flags}, + }}} + } + for _, sc := range []string{"clone", "clone3"} { + if _, err := PrivilegeDocument("p", api.ModeEnforce, section(sc)); err == nil || !strings.Contains(err.Error(), "plain fork") { + t.Errorf("%s without flags must be rejected with a clear error: %v", sc, err) + } + if _, err := PrivilegeDocument("p", api.ModeEnforce, section(sc, "CLONE_NEWNS")); err != nil { + t.Errorf("%s with flags: %v", sc, err) + } + } +} + +func TestPrivilegeDocumentExceptAndOrigin(t *testing.T) { + origin := "catalog/P1" + rule := api.PrivilegeRule{Rule: api.Rule{Name: "r", Action: api.ActionDeny, Origin: &origin}, Target: api.PrivilegeTarget{Syscall: []string{"mount"}}} + doc, err := PrivilegeDocument("p", api.ModeEnforce, api.PrivilegeSection{Default: api.ActionAllow, Rules: []api.PrivilegeRule{rule}}) + if err != nil || doc.Policy.Privilege.Rules[0].Origin == nil || *doc.Policy.Privilege.Rules[0].Origin != origin { + t.Fatalf("origin: %+v %v", doc.Policy.Privilege.Rules, err) + } + rule.Except = []api.PrivilegeException{{}} + if _, err := PrivilegeDocument("p", api.ModeEnforce, api.PrivilegeSection{Default: api.ActionAllow, Rules: []api.PrivilegeRule{rule}}); err == nil || !strings.Contains(err.Error(), "exceptions") { + t.Fatalf("an exception must be rejected: %v", err) + } +} + +// Origin presence and the default mode reach the gadget document: a missing +// origin stays nil, an explicit "" stays "". +func TestPrivilegeDocumentOriginPresenceAndDefaults(t *testing.T) { + empty := "" + section := api.PrivilegeSection{Default: api.ActionDeny, DefaultMode: api.RuleAudit, DefaultOrigin: &empty, Rules: []api.PrivilegeRule{ + {Rule: api.Rule{Name: "none", Action: api.ActionDeny}, Target: api.PrivilegeTarget{Syscall: []string{"mount"}}}, + {Rule: api.Rule{Name: "blank", Action: api.ActionDeny, Origin: &empty}, Target: api.PrivilegeTarget{Syscall: []string{"bpf"}}}, + }} + doc, err := PrivilegeDocument("p", api.ModeEnforce, section) + if err != nil { + t.Fatal(err) + } + s := doc.Policy.Privilege + if s.DefaultMode != "audit" || s.DefaultOrigin == nil || *s.DefaultOrigin != "" { + t.Fatalf("defaults: mode %q origin %v", s.DefaultMode, s.DefaultOrigin) + } + if s.Rules[0].Origin != nil || s.Rules[1].Origin == nil || *s.Rules[1].Origin != "" { + t.Fatalf("rule origins: %v %v", s.Rules[0].Origin, s.Rules[1].Origin) + } + section.DefaultOrigin = nil + if doc, _ = PrivilegeDocument("p", api.ModeEnforce, section); doc.Policy.Privilege.DefaultOrigin != nil { + t.Fatal("a missing default origin must stay missing") + } +} diff --git a/pkg/sandboxpolicy/sections.go b/pkg/sandboxpolicy/sections.go new file mode 100644 index 0000000000..41dd29e77c --- /dev/null +++ b/pkg/sandboxpolicy/sections.go @@ -0,0 +1,357 @@ +package sandboxpolicy + +import ( + "context" + "errors" + "fmt" + "sort" + "sync" + + api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" +) + +// SectionBackend drives one backend per policy section. +// +// A binding whose old and new plans have no privilege section goes straight to +// the exec backend, unchanged. Otherwise the plan is split into an exec plan +// and a privilege plan, each with its own revision. When no policy object holds +// both sections, the exec plan and its revision are what the controller gives +// the exec policies alone, so adding or removing a privilege policy does not +// restart exec. When one object holds both, its generation is part of the exec +// revision (as in the exec-only path): an edit of only its privilege section +// restarts exec once. This is a known limit; the generation must stay in the +// plan for status staleness. +// +// Commit follows the Backend contract of each section: a section that returns +// Committed=true is committed, even with a cleanup error. If the second section +// does not commit, the first section returns to its previous plan. +type SectionBackend struct { + mu sync.Mutex + exec Backend + privilege Backend // nil: privilege sections are rejected with unavailable + // unavailable says why privilege sections are rejected (privilege == nil). + unavailable error + // unqualified says why the privilege section, which enforces, must not + // report Ready (for example a kernel that is not qualified). + unqualified error + active map[string]sectionPlans +} + +type sectionPlans struct { + revision string + exec, privilege Plan + // privilegeCleanup: a privilege withdraw committed with a cleanup error; + // later calls retry it. + privilegeCleanup bool +} + +var ( + _ Backend = (*SectionBackend)(nil) + _ ProtectionAttester = (*SectionBackend)(nil) +) + +// NewSectionBackend combines the section backends. When privilege is nil, +// unavailable says why (nil means no privilege image is configured). +func NewSectionBackend(exec, privilege Backend, unavailable error) (*SectionBackend, error) { + if exec == nil { + return nil, fmt.Errorf("exec backend is required") + } + if privilege == nil && unavailable == nil { + unavailable = &EnforcementUnavailableError{Section: "privilege", Reason: "PrivilegeNotConfigured", Detail: "no privilege gadget image is configured"} + } + if privilege != nil { + unavailable = nil + } + return &SectionBackend{exec: exec, privilege: privilege, unavailable: unavailable, active: map[string]sectionPlans{}}, nil +} + +// PrivilegeUnavailable returns why privilege sections cannot be enforced, or +// why they enforce but must not report Ready. +func (s *SectionBackend) PrivilegeUnavailable() error { + if s.unavailable != nil { + return s.unavailable + } + return s.unqualified +} + +// SetPrivilegeUnqualified keeps the privilege section enforcing, but its +// policies never report Ready while err is set. Call it before the +// controller starts. +func (s *SectionBackend) SetPrivilegeUnqualified(err error) { s.unqualified = err } + +// execOnly reports whether the plans need only the exec backend. +func (p sectionPlans) execOnly() bool { return p.privilege.Revision == "" && !p.privilegeCleanup } + +// Prepare passes the plans of pods scheduled to this node, whose containers have +// not finished (running ones included), to a privilege backend that can create +// their slots before new containers start. +func (s *SectionBackend) Prepare(ctx context.Context, plans []Plan) error { + preparer, ok := s.privilege.(SlotPreparer) + if !ok { + return nil + } + return preparer.Prepare(ctx, plans) +} + +// BindPods passes the pod bindings to a privilege backend that can bind pod +// cgroups. Pod bindings exist only for the privilege section. +func (s *SectionBackend) BindPods(ctx context.Context, pods []PodBinding) error { + binder, ok := s.privilege.(PodBinder) + if !ok { + return nil + } + return binder.BindPods(ctx, pods) +} + +// BoundPodPlans returns the plans bound to pod cgroups by a privilege backend +// that reports them. Pod bindings exist only for the privilege section. +func (s *SectionBackend) BoundPodPlans() (confirmed, faulted []Plan) { + reporter, ok := s.privilege.(PodPlanReporter) + if !ok { + return nil, nil + } + return reporter.BoundPodPlans() +} + +// Exec returns the exec section backend. +func (s *SectionBackend) Exec() Backend { return s.exec } + +func split(plan Plan) (sectionPlans, error) { + s := sectionPlans{revision: plan.Revision} + type execPart struct { + Identity PolicyIdentity + Mode api.PolicyMode + RootRole string + Exec api.ExecSection + } + var execParts []execPart + for _, p := range plan.Policies { + if p.Privilege != nil { + s.privilege.Policies = append(s.privilege.Policies, p) + } + } + if len(s.privilege.Policies) == 0 { + s.exec = plan + return s, nil + } + // The privilege compiler reads each policy's identity, mode and section. + type privilegePart struct { + Identity PolicyIdentity + Mode api.PolicyMode + Privilege *api.PrivilegeSection + } + parts := make([]privilegePart, 0, len(s.privilege.Policies)) + for _, p := range s.privilege.Policies { + id := p.Identity + id.Generation = 0 // an edit of another section must not change this revision + parts = append(parts, privilegePart{id, p.Mode, p.Privilege}) + } + sort.Slice(parts, func(i, j int) bool { return parts[i].Identity.UID < parts[j].Identity.UID }) + s.privilege.Revision = digest(parts) + shared := false + for _, p := range plan.Policies { + if p.Privilege != nil && p.Exec.Default != "" { + shared = true + } + } + if !shared { + // No policy object carries both sections: the exec plan is the plan + // of the other policies, with the revision the controller gives + // that plan alone. Adding or removing a privilege policy then keeps + // the exec revision, so exec does not restart. + for _, p := range plan.Policies { + if p.Privilege == nil { + s.exec.Policies = append(s.exec.Policies, p) + } + } + if len(s.exec.Policies) > 0 { + s.exec.Revision = digest(s.exec.Policies) + } + return s, nil + } + for _, p := range plan.Policies { + if p.Exec.Default == "" { + continue + } + e := p + e.Privilege = nil + s.exec.Policies = append(s.exec.Policies, e) + execParts = append(execParts, execPart{p.Identity, p.Mode, p.RootRole, p.Exec}) + } + if len(execParts) > 0 { + s.exec.Revision = digest(execParts) + } + return s, nil +} + +type sectionResult struct { + committed, unconfirmed bool + err error +} + +// apply moves one section from old to want. +func apply(ctx context.Context, backend Backend, binding Binding, old, want Plan) sectionResult { + switch { + case want.Revision == old.Revision: + if want.Revision == "" { + return sectionResult{committed: true} + } + fallthrough // same plan: retry cleanup and confirm protection + case want.Revision != "": + r, err := backend.Replace(ctx, binding, want) + committed := r.Committed && r.ActiveRevision == want.Revision + if !committed && err == nil { + err = fmt.Errorf("replace not committed") + } + return sectionResult{committed, r.ProtectionUnconfirmed, err} + default: + r, err := backend.Withdraw(ctx, binding) + if !r.Committed && err == nil { + err = fmt.Errorf("withdraw not committed") + } + return sectionResult{r.Committed, r.ProtectionUnconfirmed, err} + } +} + +func (s *SectionBackend) Replace(ctx context.Context, binding Binding, plan Plan) (ReplaceResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + k := bindingKey(binding) + old := s.active[k] + want, err := split(plan) + if err != nil { + return ReplaceResult{}, err + } + if old.execOnly() && want.execOnly() { + r, err := s.exec.Replace(ctx, binding, plan) + if r.Committed { + s.active[k] = want + } + return r, err + } + if s.privilege == nil { + return ReplaceResult{}, s.unavailable + } + + // Two phases: first prepare the privilege slot, which can fail and + // changes no binding; then commit exec, then make the live privilege + // bind. A failed prepare leaves both sections on their last-good plan. + if preparer, ok := s.privilege.(ReplacePreparer); ok && want.privilege.Revision != "" && want.privilege.Revision != old.privilege.Revision { + if err := preparer.PrepareReplace(ctx, binding, want.privilege); err != nil { + return ReplaceResult{}, fmt.Errorf("privilege section: %w", err) + } + } + oldPrivilege := old.privilege + if old.privilegeCleanup && oldPrivilege.Revision == "" { + oldPrivilege.Revision = "cleanup" // forces a privilege Withdraw retry + } + if want.privilege.Revision == "" { + // The privilege section is removed: withdraw it first. A withdraw + // that does not commit (the instance is not ready) changes nothing + // and is retried, with no exec commit and rollback each time. + p := apply(ctx, s.privilege, binding, oldPrivilege, want.privilege) + if !p.committed { + return ReplaceResult{ProtectionUnconfirmed: p.unconfirmed}, fmt.Errorf("privilege section: %w", p.err) + } + e := apply(ctx, s.exec, binding, old.exec, want.exec) + if !e.committed { + // Exec keeps its last-good plan; the privilege removal stays. + // The binding now runs neither the last-good nor the desired + // plan, so its protection is unconfirmed. + // No revision: Attest fails until a Replace commits. + old.privilege, old.privilegeCleanup, old.revision = Plan{}, p.err != nil, "" + s.active[k] = old + return ReplaceResult{ProtectionUnconfirmed: true}, fmt.Errorf("exec section: %w", e.err) + } + want.privilegeCleanup = p.err != nil + s.active[k] = want + return ReplaceResult{Committed: true, ActiveRevision: plan.Revision, ProtectionUnconfirmed: e.unconfirmed || p.unconfirmed}, errors.Join(e.err, p.err) + } + e := apply(ctx, s.exec, binding, old.exec, want.exec) + if !e.committed { + return ReplaceResult{ProtectionUnconfirmed: e.unconfirmed}, fmt.Errorf("exec section: %w", e.err) + } + p := apply(ctx, s.privilege, binding, oldPrivilege, want.privilege) + if !p.committed { + // The live bind failed after a successful prepare (identity check + // or map write): return exec to its previous plan. + back := apply(ctx, s.exec, binding, want.exec, old.exec) + return ReplaceResult{ProtectionUnconfirmed: e.unconfirmed || p.unconfirmed || back.unconfirmed || !back.committed}, + errors.Join(fmt.Errorf("privilege section: %w", p.err), back.err) + } + s.active[k] = want + return ReplaceResult{Committed: true, ActiveRevision: plan.Revision, ProtectionUnconfirmed: e.unconfirmed || p.unconfirmed}, errors.Join(e.err, p.err) +} + +func (s *SectionBackend) Withdraw(ctx context.Context, binding Binding) (WithdrawResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + k := bindingKey(binding) + old := s.active[k] + if old.execOnly() { + r, err := s.exec.Withdraw(ctx, binding) + if r.Committed { + delete(s.active, k) + } + return r, err + } + e, eerr := s.exec.Withdraw(ctx, binding) + p, perr := s.privilege.Withdraw(ctx, binding) + result := WithdrawResult{ + Committed: e.Committed && p.Committed, + // One section removed and the other not: the old protection is gone. + ProtectionUnconfirmed: e.ProtectionUnconfirmed || p.ProtectionUnconfirmed || e.Committed != p.Committed, + } + switch { + case result.Committed && perr != nil: + s.active[k] = sectionPlans{privilegeCleanup: true} + case result.Committed: + delete(s.active, k) + case e.Committed: + old.exec = Plan{} + s.active[k] = old + } + return result, errors.Join(eerr, perr) +} + +// Attest checks each section that is active for the binding. +func (s *SectionBackend) Attest(ctx context.Context, binding Binding, revision string) error { + s.mu.Lock() + plans, ok := s.active[bindingKey(binding)] + s.mu.Unlock() + if !ok || plans.execOnly() { + if a, isAttester := s.exec.(ProtectionAttester); isAttester { + return a.Attest(ctx, binding, revision) + } + return fmt.Errorf("exec backend cannot attest") + } + if plans.revision != revision { + return fmt.Errorf("revision %q is not active", revision) + } + var errs []error + for _, section := range []struct { + backend Backend + plan Plan + }{{s.exec, plans.exec}, {s.privilege, plans.privilege}} { + if section.plan.Revision == "" { + continue + } + a, isAttester := section.backend.(ProtectionAttester) + if !isAttester { + errs = append(errs, fmt.Errorf("backend cannot attest")) + continue + } + errs = append(errs, a.Attest(ctx, binding, section.plan.Revision)) + } + return errors.Join(errs...) +} + +func (s *SectionBackend) Close(ctx context.Context) error { + var errs []error + for _, backend := range []Backend{s.exec, s.privilege} { + if c, ok := backend.(interface{ Close(context.Context) error }); ok { + errs = append(errs, c.Close(ctx)) + } + } + return errors.Join(errs...) +} diff --git a/pkg/sandboxpolicy/sections_test.go b/pkg/sandboxpolicy/sections_test.go new file mode 100644 index 0000000000..1159f4f36d --- /dev/null +++ b/pkg/sandboxpolicy/sections_test.go @@ -0,0 +1,498 @@ +package sandboxpolicy + +import ( + "context" + "errors" + "strings" + "testing" + + api "github.com/kubescape/storage/pkg/apis/sandbox/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +func privilegeSection(def api.Action) *api.PrivilegeSection { + return &api.PrivilegeSection{Default: def, Rules: []api.PrivilegeRule{{Rule: api.Rule{Name: "no-mount", Action: api.ActionDeny}, Target: api.PrivilegeTarget{Syscall: []string{"mount"}}}}} +} + +func sectionPlan(revision string, policies ...CompiledPolicy) Plan { + return Plan{Revision: revision, Policies: policies} +} + +func TestSectionBackendExecOnlyPlanIsUnchanged(t *testing.T) { + exec, privilege := &fakeBackend{plans: map[string]Plan{}}, &fakeBackend{plans: map[string]Plan{}} + s, err := NewSectionBackend(exec, privilege, nil) + if err != nil { + t.Fatal(err) + } + plan := sectionPlan("aggregate", CompiledPolicy{Revision: "p1", Exec: api.ExecSection{Default: api.ActionDeny}}) + r, err := s.Replace(context.Background(), Binding{ContainerID: "c"}, plan) + if err != nil || !r.Committed || r.ActiveRevision != "aggregate" { + t.Fatalf("replace: %+v %v", r, err) + } + if got := exec.plans[bindingKey(Binding{ContainerID: "c"})]; got.Revision != "aggregate" { + t.Fatalf("exec got %+v", got) + } + if privilege.replacements+privilege.withdrawals != 0 { + t.Fatal("privilege backend must not be called for an exec-only plan") + } + if err := s.Attest(context.Background(), Binding{ContainerID: "c"}, "aggregate"); err != nil { + t.Fatal(err) + } +} + +func TestSectionBackendSplitsAndKeepsUnchangedSection(t *testing.T) { + exec, privilege := &fakeBackend{plans: map[string]Plan{}}, &fakeBackend{plans: map[string]Plan{}} + s, _ := NewSectionBackend(exec, privilege, nil) + b := Binding{ContainerID: "c"} + execPolicy := CompiledPolicy{Revision: "e", Exec: api.ExecSection{Default: api.ActionDeny}} + privPolicy := CompiledPolicy{Revision: "p", Privilege: privilegeSection(api.ActionAllow)} + if _, err := s.Replace(context.Background(), b, sectionPlan("r1", execPolicy, privPolicy)); err != nil { + t.Fatal(err) + } + if len(exec.plans[bindingKey(b)].Policies) != 1 || exec.plans[bindingKey(b)].Policies[0].Privilege != nil { + t.Fatalf("exec plan %+v", exec.plans[bindingKey(b)]) + } + if len(privilege.plans[bindingKey(b)].Policies) != 1 { + t.Fatalf("privilege plan %+v", privilege.plans[bindingKey(b)]) + } + // An exec-only change keeps the privilege revision: no privilege restart. + execPolicy.Revision = "e2" + execPolicy.Exec.Default = api.ActionAllow + if _, err := s.Replace(context.Background(), b, sectionPlan("r2", execPolicy, privPolicy)); err != nil { + t.Fatal(err) + } + if exec.replacements != 2 || privilege.replacements != 1 { + t.Fatalf("exec %d privilege %d replacements", exec.replacements, privilege.replacements) + } + // Dropping the privilege policy withdraws only the privilege section. + if _, err := s.Replace(context.Background(), b, sectionPlan("r3", execPolicy)); err != nil { + t.Fatal(err) + } + if privilege.withdrawals != 1 || exec.withdrawals != 0 { + t.Fatalf("withdrawals exec %d privilege %d", exec.withdrawals, privilege.withdrawals) + } +} + +func TestSectionBackendPrivilegeFailureRestoresExec(t *testing.T) { + exec, privilege := &fakeBackend{plans: map[string]Plan{}}, &fakeBackend{plans: map[string]Plan{}, fail: true} + s, _ := NewSectionBackend(exec, privilege, nil) + b := Binding{ContainerID: "c"} + execPolicy := CompiledPolicy{Revision: "e", Exec: api.ExecSection{Default: api.ActionDeny}} + r, err := s.Replace(context.Background(), b, sectionPlan("r1", execPolicy, CompiledPolicy{Revision: "p", Privilege: privilegeSection(api.ActionAllow)})) + if err == nil || r.Committed { + t.Fatalf("expected failure, got %+v", r) + } + if _, ok := exec.plans[bindingKey(b)]; ok || exec.withdrawals != 1 { + t.Fatalf("exec must return to its previous (empty) state: %+v withdrawals=%d", exec.plans, exec.withdrawals) + } +} + +func TestSectionBackendRejectsPrivilegeWithoutImage(t *testing.T) { + s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, nil, nil) + if _, err := s.Replace(context.Background(), Binding{}, sectionPlan("r", CompiledPolicy{Privilege: privilegeSection(api.ActionDeny)})); err == nil { + t.Fatal("privilege must be rejected without a privilege backend") + } +} + +func TestCompileKeepsPrivilegeSection(t *testing.T) { + spec := execSpec() + spec.Privilege = privilegeSection(api.ActionAllow) + got, err := Compile(context.Background(), PolicyIdentity{}, spec, api.PolicyClasses{}, Binding{}, nil) + if err != nil || got.Privilege == nil || got.Privilege.Rules[0].Name != "no-mount" { + t.Fatalf("compile: %+v %v", got.Privilege, err) + } + spec.Privilege.Default = "maybe" + if _, err := Compile(context.Background(), PolicyIdentity{}, spec, api.PolicyClasses{}, Binding{}, nil); err == nil { + t.Fatal("invalid privilege default accepted") + } +} + +func TestSectionBackendKeepsCommittedCleanupErrors(t *testing.T) { + // Exec-only: the exec backend result passes through unchanged. + exec := &fakeBackend{plans: map[string]Plan{}, postCommit: true} + s, _ := NewSectionBackend(exec, &fakeBackend{plans: map[string]Plan{}}, nil) + b := Binding{ContainerID: "c"} + r, err := s.Replace(context.Background(), b, sectionPlan("r1", CompiledPolicy{Exec: api.ExecSection{Default: api.ActionDeny}})) + if err == nil || !r.Committed || r.ActiveRevision != "r1" { + t.Fatalf("exec-only cleanup error must stay committed: %+v %v", r, err) + } + // With privilege: a committed section with a cleanup error does not roll back. + privilege := &fakeBackend{plans: map[string]Plan{}, postCommit: true} + s, _ = NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege, nil) + r, err = s.Replace(context.Background(), b, sectionPlan("r2", CompiledPolicy{Privilege: privilegeSection(api.ActionAllow)})) + if err == nil || !r.Committed || r.ActiveRevision != "r2" { + t.Fatalf("privilege cleanup error must stay committed: %+v %v", r, err) + } +} + +func TestSectionBackendPrivilegeChangeKeepsExecRevision(t *testing.T) { + exec, privilege := &fakeBackend{plans: map[string]Plan{}}, &fakeBackend{plans: map[string]Plan{}} + s, _ := NewSectionBackend(exec, privilege, nil) + b := Binding{ContainerID: "c"} + p := CompiledPolicy{Revision: "v1", Exec: api.ExecSection{Default: api.ActionDeny}, Privilege: privilegeSection(api.ActionAllow)} + if _, err := s.Replace(context.Background(), b, sectionPlan("r1", p)); err != nil { + t.Fatal(err) + } + p.Revision = "v2" // the compiler's policy revision changes with the privilege section + p.Privilege = privilegeSection(api.ActionDeny) + if _, err := s.Replace(context.Background(), b, sectionPlan("r2", p)); err != nil { + t.Fatal(err) + } + if exec.replacements != 1 || privilege.replacements != 2 { + t.Fatalf("exec %d privilege %d replacements", exec.replacements, privilege.replacements) + } +} + +func TestSectionBackendSendsAllPrivilegePolicies(t *testing.T) { + privilege := &fakeBackend{plans: map[string]Plan{}} + s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege, nil) + b := Binding{ContainerID: "c"} + p1 := CompiledPolicy{Identity: PolicyIdentity{UID: "1"}, Privilege: privilegeSection(api.ActionAllow)} + p2 := CompiledPolicy{Identity: PolicyIdentity{UID: "2"}, Privilege: privilegeSection(api.ActionDeny)} + if _, err := s.Replace(context.Background(), b, sectionPlan("r", p1, p2)); err != nil { + t.Fatal(err) + } + if got := privilege.plans[bindingKey(b)]; len(got.Policies) != 2 { + t.Fatalf("privilege plan %+v", got) + } +} + +type preparingBackend struct { + fakeBackend + prepared []Plan +} + +func (p *preparingBackend) Prepare(_ context.Context, plans []Plan) error { + p.prepared = plans + return nil +} + +func TestSectionBackendPassesPrepare(t *testing.T) { + privilege := &preparingBackend{fakeBackend: fakeBackend{plans: map[string]Plan{}}} + s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege, nil) + if err := s.Prepare(context.Background(), []Plan{sectionPlan("r", CompiledPolicy{Privilege: privilegeSection(api.ActionDeny)})}); err != nil || len(privilege.prepared) != 1 { + t.Fatalf("prepare: %v %v", err, privilege.prepared) + } +} + +func TestSectionBackendModeChangeReachesPrivilege(t *testing.T) { + privilege := &fakeBackend{plans: map[string]Plan{}} + s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege, nil) + b := Binding{ContainerID: "c"} + p := CompiledPolicy{Mode: api.ModeAudit, Privilege: privilegeSection(api.ActionAllow)} + if _, err := s.Replace(context.Background(), b, sectionPlan("r1", p)); err != nil { + t.Fatal(err) + } + p.Mode = api.ModeEnforce + if _, err := s.Replace(context.Background(), b, sectionPlan("r2", p)); err != nil { + t.Fatal(err) + } + if privilege.replacements != 2 { + t.Fatalf("mode change must replace the privilege instance, replacements=%d", privilege.replacements) + } +} + +func TestSectionBackendRetriesPrivilegeCleanup(t *testing.T) { + privilege := &fakeBackend{plans: map[string]Plan{}} + s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege, nil) + b := Binding{ContainerID: "c"} + if _, err := s.Replace(context.Background(), b, sectionPlan("r1", CompiledPolicy{Privilege: privilegeSection(api.ActionAllow)})); err != nil { + t.Fatal(err) + } + privilege.withdrawPostCommit = true + r, err := s.Withdraw(context.Background(), b) + if err == nil || !r.Committed { + t.Fatalf("committed withdraw with cleanup error expected: %+v %v", r, err) + } + privilege.withdrawPostCommit = false + if r, err = s.Withdraw(context.Background(), b); err != nil || !r.Committed || privilege.withdrawals != 2 { + t.Fatalf("retry must reach the privilege backend: %+v %v withdrawals=%d", r, err, privilege.withdrawals) + } + if _, ok := s.active[bindingKey(b)]; ok { + t.Fatal("state must be gone after a clean retry") + } +} + +func TestSectionBackendPartialWithdrawIsUnconfirmed(t *testing.T) { + privilege := &fakeBackend{plans: map[string]Plan{}} + s, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, privilege, nil) + b := Binding{ContainerID: "c"} + p := CompiledPolicy{Exec: api.ExecSection{Default: api.ActionDeny}, Privilege: privilegeSection(api.ActionAllow)} + if _, err := s.Replace(context.Background(), b, sectionPlan("r1", p)); err != nil { + t.Fatal(err) + } + privilege.fail = true + if r, _ := s.Withdraw(context.Background(), b); r.Committed || !r.ProtectionUnconfirmed { + t.Fatalf("exec removed but privilege not: %+v", r) + } +} + +func TestControllerPrepareScheduled(t *testing.T) { + ctx := context.Background() + withPrivilege := policy("with-privilege", "ns") + withPrivilege.Spec.Privilege = privilegeSection(api.ActionDeny) + execOnly := policy("exec-only", "ns") + c, _, _, _ := setup(t, withPrivilege, execOnly) + preparer := &preparingBackend{fakeBackend: fakeBackend{plans: map[string]Plan{}}} + c.backend = preparer + if err := c.InitialSync(ctx); err != nil { + t.Fatal(err) + } + template := binding() + template.ContainerID, template.CgroupID = "", 0 // a scheduled pod: no runtime identity yet + other := template + other.Namespace = "elsewhere" + if err := c.PrepareScheduled(ctx, []Binding{template, other}); err != nil { + t.Fatal(err) + } + if len(preparer.prepared) != 1 || len(preparer.prepared[0].Policies) != 1 || preparer.prepared[0].Policies[0].Identity.Name != "with-privilege" { + t.Fatalf("prepared %+v", preparer.prepared) + } +} + +func TestPrivilegeUnavailableIsReportedAndRefused(t *testing.T) { + ctx := context.Background() + p := policy("with-privilege", "ns") + p.Spec.Privilege = privilegeSection(api.ActionDeny) + c, _, w, _ := setup(t, p) + lsm := &EnforcementUnavailableError{Section: "privilege", Reason: "BPFLSMInactive", Detail: "BPF LSM is not active"} + sections, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, nil, lsm) + c.backend = sections + if err := c.InitialSync(ctx); err != nil && !errors.Is(err, lsm) { + t.Fatalf("initial sync: %v", err) + } + report := w.reports[string(p.UID)] + if report == nil || len(report.Status.Conditions) == 0 { + t.Fatal("no status report") + } + ready := report.Status.Conditions[0] + if ready.Status != metav1.ConditionFalse || ready.Reason != "BPFLSMInactive" { + t.Fatalf("ready condition %+v", ready) + } + if report.Status.Tier != "none" { + t.Fatalf("tier %q, want none", report.Status.Tier) + } + if _, err := sections.Replace(ctx, Binding{ContainerID: "c"}, sectionPlan("r", CompiledPolicy{Privilege: privilegeSection(api.ActionDeny)})); !errors.Is(err, lsm) { + t.Fatalf("privilege binding must be refused: %v", err) + } +} + +// The controller gives a plan the digest of its policies as revision. Adding a +// privilege policy to an exec-only binding, and removing it again, must keep +// the exec revision, so exec does not restart. +func TestSectionBackendSeparatePrivilegePolicyKeepsExecRevision(t *testing.T) { + exec, privilege := &fakeBackend{plans: map[string]Plan{}}, &fakeBackend{plans: map[string]Plan{}} + s, _ := NewSectionBackend(exec, privilege, nil) + b := Binding{ContainerID: "c"} + execPolicy := CompiledPolicy{Identity: PolicyIdentity{UID: "e", Generation: 3}, Revision: "e", Exec: api.ExecSection{Default: api.ActionDeny}} + privPolicy := CompiledPolicy{Identity: PolicyIdentity{UID: "p", Generation: 1}, Revision: "p", Privilege: privilegeSection(api.ActionAllow)} + plan := func(policies ...CompiledPolicy) Plan { return Plan{Revision: digest(policies), Policies: policies} } + steps := []Plan{plan(execPolicy), plan(execPolicy, privPolicy), plan(execPolicy)} + privPolicy.Identity.Generation = 2 // an edit of the privilege policy object + privPolicy.Privilege = privilegeSection(api.ActionDeny) + steps = append(steps[:2], plan(execPolicy, privPolicy), steps[2]) + for i, p := range steps { + r, err := s.Replace(context.Background(), b, p) + if err != nil || !r.Committed || r.ActiveRevision != p.Revision { + t.Fatalf("step %d: %+v %v", i, r, err) + } + } + if exec.replacements != 1 { + t.Fatalf("exec restarted: %d replacements", exec.replacements) + } + if privilege.replacements != 2 || privilege.withdrawals != 1 { + t.Fatalf("privilege %d replacements %d withdrawals", privilege.replacements, privilege.withdrawals) + } +} + +func TestControllerPodPlans(t *testing.T) { + ctx := context.Background() + all := policy("all", "ns") + all.Spec.Privilege = privilegeSection(api.ActionDeny) + app := policy("app-only", "ns") + app.Spec.Containers = []string{"app"} + app.Spec.Privilege = privilegeSection(api.ActionAllow) + c, _, _, _ := setup(t, all, app) + if err := c.InitialSync(ctx); err != nil { + t.Fatal(err) + } + container := func(pod types.UID, name, kind string) Binding { + b := binding() + b.ContainerID, b.CgroupID, b.PodUID, b.ContainerName, b.Kind = "", 0, pod, name, kind + return b + } + elsewhere := container("other-ns", "web", "regular") + elsewhere.Namespace = "elsewhere" + plans, ok := c.PodPlans([]Binding{ + container("uniform", "web", "regular"), container("uniform", "db", "regular"), + container("init", "web", "regular"), container("init", "setup", "init"), + container("mixed", "web", "regular"), container("mixed", "app", "regular"), + elsewhere, + }) + if !ok || len(plans) != 1 || len(plans["uniform"].Policies) != 1 || plans["uniform"].Policies[0].Identity.Name != "all" { + t.Fatalf("only the pod whose containers all get the same policies has a pod plan: %+v", plans) + } +} + +// An invalid policy that may select the pod makes its input incomplete: the +// pod gets an empty plan (keep the current binding), not a removal. +func TestControllerPodPlansKeepOnIncompleteInput(t *testing.T) { + ctx := context.Background() + priv := policy("priv", "ns") + priv.Spec.Privilege = privilegeSection(api.ActionDeny) + broken := policy("broken", "ns") + broken.Spec.Containers = nil // invalid: containers must not be empty + c, _, _, _ := setup(t, priv, broken) + _ = c.InitialSync(ctx) + pod := binding() + pod.ContainerID, pod.CgroupID, pod.PodUID = "", 0, "pod" + plans, ok := c.PodPlans([]Binding{pod}) + plan, present := plans["pod"] + if !ok || !present || len(plan.Policies) != 0 { + t.Fatalf("incomplete input must keep the pod binding (empty plan): %+v", plans) + } +} + +func TestControllerPodPlansNeedContainerEligibility(t *testing.T) { + ctx := context.Background() + priv := policy("priv", "ns") + priv.Spec.Privilege = privilegeSection(api.ActionDeny) + // A second policy that selects the same container but depends on a + // ContainerProfile: the container binding can fail, so no pod binding. + sourced := policy("sourced", "ns") + sourced.Spec.Selector.PodSelector = metav1.LabelSelector{MatchLabels: map[string]string{"profile": "yes"}} + sourced.Spec.Sources = []api.PolicySource{{ContainerProfile: api.ContainerProfileReference{Name: "profile"}, Include: []string{"execs"}}} + c, _, _, _ := setup(t, priv, sourced) + if err := c.InitialSync(ctx); err != nil && !strings.Contains(err.Error(), "ContainerProfile") { + t.Fatal(err) + } + plain := binding() + plain.ContainerID, plain.CgroupID, plain.PodUID = "", 0, "plain" + profiled := plain + profiled.PodUID = "profiled" + profiled.PodLabels = map[string]string{"app": "test", "profile": "yes"} + plans, ok := c.PodPlans([]Binding{plain, profiled}) + if !ok || len(plans) != 1 || len(plans["plain"].Policies) != 1 { + t.Fatalf("only the pod without a sourced policy has a pod plan: %+v", plans) + } +} + +// Every section enforces through BPF LSM. Without it, an exec-only policy must +// not report Ready or a bpf-lsm tier either. +func TestNodeWithoutBPFLSMReportsTierNone(t *testing.T) { + ctx := context.Background() + execOnly := policy("exec-only", "ns") + off := policy("off", "ns") + off.Spec.Mode = api.ModeOff + c, _, w, _ := setup(t, execOnly, off) + lsm := &EnforcementUnavailableError{Section: "sandbox policy", Reason: "BPFLSMInactive", Detail: "BPF LSM is not active"} + c.options.EnforcementUnavailable = lsm + if err := c.InitialSync(ctx); err != nil && !errors.Is(err, lsm) { + t.Fatalf("initial sync: %v", err) + } + enabled := w.reports[string(execOnly.UID)] + if enabled == nil || enabled.Status.Tier != "none" || enabled.Status.Conditions[0].Status != metav1.ConditionFalse || enabled.Status.Conditions[0].Reason != "BPFLSMInactive" { + t.Fatalf("exec-only report %+v", enabled) + } + disabled := w.reports[string(off.UID)] + if disabled == nil || disabled.Status.Tier != "none" || disabled.Status.Conditions[0].Status != metav1.ConditionTrue { + t.Fatalf("off policy report %+v", disabled) + } +} + +func TestNodeWithBPFLSMReportsTierBPFLSM(t *testing.T) { + c, _, w, _ := setup(t, policy("exec-only", "ns")) + if err := c.InitialSync(context.Background()); err != nil { + t.Fatal(err) + } + if r := w.reports["exec-only"]; r == nil || r.Status.Tier != "bpf-lsm" { + t.Fatalf("report %+v", r) + } +} + +type replacePreparingBackend struct { + fakeBackend + prepareErr error + prepared []string +} + +func (b *replacePreparingBackend) PrepareReplace(_ context.Context, _ Binding, plan Plan) error { + b.prepared = append(b.prepared, plan.Revision) + return b.prepareErr +} + +// The privilege slot is prepared before exec commits: a failed prepare leaves +// both sections on their last-good plan, with no exec rollback. +func TestSectionBackendPreparesPrivilegeBeforeExec(t *testing.T) { + exec := &fakeBackend{plans: map[string]Plan{}} + privilege := &replacePreparingBackend{fakeBackend: fakeBackend{plans: map[string]Plan{}}, prepareErr: errors.New("slot does not fit")} + s, _ := NewSectionBackend(exec, privilege, nil) + b := Binding{ContainerID: "c"} + execPolicy := CompiledPolicy{Revision: "e", Exec: api.ExecSection{Default: api.ActionDeny}} + privPolicy := CompiledPolicy{Revision: "p", Privilege: privilegeSection(api.ActionAllow)} + r, err := s.Replace(context.Background(), b, sectionPlan("r1", execPolicy, privPolicy)) + if err == nil || r.Committed || r.ProtectionUnconfirmed { + t.Fatalf("failed prepare: %+v %v", r, err) + } + if exec.replacements+exec.withdrawals != 0 || privilege.replacements != 0 { + t.Fatalf("a failed prepare must change no section: exec %d/%d privilege %d", exec.replacements, exec.withdrawals, privilege.replacements) + } + privilege.prepareErr = nil + if r, err := s.Replace(context.Background(), b, sectionPlan("r1", execPolicy, privPolicy)); err != nil || !r.Committed { + t.Fatalf("replace: %+v %v", r, err) + } + // An unchanged privilege revision needs no prepare. + execPolicy.Revision = "e2" + if _, err := s.Replace(context.Background(), b, sectionPlan("r2", execPolicy, privPolicy)); err != nil { + t.Fatal(err) + } + if len(privilege.prepared) != 2 { + t.Fatalf("prepares %v", privilege.prepared) + } +} + +type pendingWithdrawBackend struct { + fakeBackend + pending bool +} + +func (b *pendingWithdrawBackend) Withdraw(ctx context.Context, binding Binding) (WithdrawResult, error) { + if b.pending { + return WithdrawResult{ProtectionUnconfirmed: true}, errors.New("unbind pending") + } + return b.fakeBackend.Withdraw(ctx, binding) +} + +// Removing the privilege section withdraws it before exec commits: a pending +// withdraw changes no exec plan, so exec is not reloaded and rolled back on +// each retry. +func TestSectionBackendPendingPrivilegeRemovalKeepsExec(t *testing.T) { + exec := &fakeBackend{plans: map[string]Plan{}} + privilege := &pendingWithdrawBackend{fakeBackend: fakeBackend{plans: map[string]Plan{}}} + s, _ := NewSectionBackend(exec, privilege, nil) + b := Binding{ContainerID: "c"} + execPolicy := CompiledPolicy{Revision: "e", Exec: api.ExecSection{Default: api.ActionDeny}} + privPolicy := CompiledPolicy{Revision: "p", Privilege: privilegeSection(api.ActionAllow)} + if _, err := s.Replace(context.Background(), b, sectionPlan("r1", execPolicy, privPolicy)); err != nil { + t.Fatal(err) + } + privilege.pending = true + execPolicy.Revision = "e2" + execPolicy.Exec.Default = api.ActionAllow + for i := 0; i < 2; i++ { + if r, err := s.Replace(context.Background(), b, sectionPlan("r2", execPolicy)); err == nil || r.Committed { + t.Fatalf("pending removal: %+v %v", r, err) + } + } + if exec.replacements != 1 || exec.withdrawals != 0 { + t.Fatalf("exec must not change while the removal is pending: %d replacements, %d withdrawals", exec.replacements, exec.withdrawals) + } + privilege.pending = false + if r, err := s.Replace(context.Background(), b, sectionPlan("r2", execPolicy)); err != nil || !r.Committed { + t.Fatalf("retry: %+v %v", r, err) + } + if exec.replacements != 2 { + t.Fatalf("exec commits after the removal: %d", exec.replacements) + } +} diff --git a/pkg/sandboxpolicy/status.go b/pkg/sandboxpolicy/status.go index bc7536faaf..f785a68ec0 100644 --- a/pkg/sandboxpolicy/status.go +++ b/pkg/sandboxpolicy/status.go @@ -30,6 +30,21 @@ func (c *Controller) report(p desiredPolicy, failure error) *api.SandboxPolicyNo var cleanup *cleanupError cleanupPending := errors.As(failure, &cleanup) revisions := make(map[string]bool) + count := func(compiled CompiledPolicy) { + active++ + // The oldest generation still installed is the one this node can vouch for. + if status.AppliedGeneration == 0 || compiled.Identity.Generation < status.AppliedGeneration { + status.AppliedGeneration = compiled.Identity.Generation + } + entry := fmt.Sprintf("generation=%d", compiled.Identity.Generation) + if compiled.Revision != "" { // pod plans carry no revision + entry += " revision=" + compiled.Revision + } + revisions[entry] = true + if compiled.Identity.Generation != p.identity.Generation { + stale = true + } + } for _, installed := range c.active { for _, compiled := range installed.plan.Policies { if compiled.Identity.UID != p.identity.UID { @@ -39,10 +54,25 @@ func (c *Controller) report(p desiredPolicy, failure error) *api.SandboxPolicyNo unconfirmed = true continue } - active++ - revisions[fmt.Sprintf("generation=%d revision=%s", compiled.Identity.Generation, compiled.Revision)] = true - if compiled.Identity.Generation != p.identity.Generation { - stale = true + count(compiled) + } + } + // Pod cgroup bindings enforce too, even when no container binding does, + // but only where the witness counters confirm it. + if reporter, ok := c.backend.(PodPlanReporter); ok { + confirmed, faulted := reporter.BoundPodPlans() + for _, plan := range confirmed { + for _, compiled := range plan.Policies { + if compiled.Identity.UID == p.identity.UID { + count(compiled) + } + } + } + for _, plan := range faulted { + for _, compiled := range plan.Policies { + if compiled.Identity.UID == p.identity.UID { + unconfirmed = true + } } } } @@ -53,6 +83,23 @@ func (c *Controller) report(p desiredPolicy, failure error) *api.SandboxPolicyNo if p.spec.Mode == api.ModeOff { message = "Policy is off" } + // Without BPF LSM the programs attach but never run: nothing enforces or + // audits, so the tier is none and an enabled policy is never Ready. + if c.options.EnforcementUnavailable != nil { + status.Tier = "none" + status.AppliedGeneration = 0 // installed, but no generation is enforced + if p.spec.Mode != api.ModeOff { + failure = errors.Join(failure, c.options.EnforcementUnavailable) + } + } else if p.spec.Privilege != nil && p.spec.Mode != api.ModeOff { + if err := c.privilegeUnavailable(); err != nil { + // Last-good exec bindings still enforce through BPF LSM. + if active == 0 { + status.Tier = "none" + } + failure = errors.Join(failure, err) + } + } var uncertain *protectionError unconfirmed = unconfirmed || errors.As(failure, &uncertain) if unconfirmed { @@ -64,8 +111,11 @@ func (c *Controller) report(p desiredPolicy, failure error) *api.SandboxPolicyNo reason = "NotReady" message = "Desired revision rejected; no last-good enforcement exists" if active > 0 { + // Stale only when installed bindings lag the desired generation. message = "Desired revision rejected; retaining existing last-good bindings; new bindings may be unprotected" - stale = true + if !generationStale { + message = "Desired revision applied in part; installed bindings enforce it, but some bindings failed; see error" + } } if cleanupPending { reason = "CleanupPending" @@ -78,6 +128,12 @@ func (c *Controller) report(p desiredPolicy, failure error) *api.SandboxPolicyNo message = "Binding withdrawal committed; enforcement removed for that binding, resource cleanup pending" stale = generationStale } + // A node-wide cause names the node, not the first section it hit. + if r, m, ok := unavailableReason(c.options.EnforcementUnavailable); ok && p.spec.Mode != api.ModeOff { + reason, message = r, m + } else if r, m, ok := unavailableReason(failure); ok { + reason, message = r, m + } if unconfirmed { reason = "ProtectionUnconfirmed" message = "Owned binding protection cannot be confirmed; workload may be unprotected; ownership retained only for recovery/cleanup" diff --git a/pkg/sandboxpolicy/status_test.go b/pkg/sandboxpolicy/status_test.go index 2e9be7575e..7d03c98510 100644 --- a/pkg/sandboxpolicy/status_test.go +++ b/pkg/sandboxpolicy/status_test.go @@ -22,6 +22,9 @@ func TestStatusIdentityBoundsAndUnprotected(t *testing.T) { if !r.Status.Truncated || len(r.Status.Error) != api.MaxStatusErrorLength || r.Status.Conditions[0].Status != metav1.ConditionFalse || !strings.Contains(r.Status.Conditions[0].Message, "no last-good") { t.Fatalf("incorrect failure: %+v", r.Status) } + if r.Status.AppliedGeneration != 0 { + t.Fatalf("applied generation %d with no installed binding", r.Status.AppliedGeneration) + } } func parseTypedPolicy(t *testing.T, p *api.SandboxPolicy) desiredPolicy { @@ -74,3 +77,161 @@ func TestDynamicStatusWriterCreateHeartbeatAndIdentity(t *testing.T) { } } } + +func TestAppliedGenerationIsOldestConfirmedBinding(t *testing.T) { + c, _, _, _ := setup(t) + p := parseTypedPolicy(t, policy("policy", "ns")) + p.identity.Generation = 3 + installedAt := func(container string, generation int64, unconfirmed bool) { + b := binding() + b.ContainerID = container + c.active[bindingKey(b)] = installed{binding: b, protectionUnconfirmed: unconfirmed, plan: Plan{Policies: []CompiledPolicy{{Identity: PolicyIdentity{UID: p.identity.UID, Generation: generation}, Revision: container}}}} + } + installedAt("new", 3, false) + installedAt("old", 2, false) + installedAt("unconfirmed", 1, true) // cannot vouch for it + if r := c.report(p, nil); r.Status.AppliedGeneration != 2 || r.Status.Tier != "bpf-lsm" { + t.Fatalf("status %+v, want applied generation 2", r.Status) + } + c.options.EnforcementUnavailable = &EnforcementUnavailableError{Section: "sandbox policy", Reason: "BPFLSMInactive", Detail: "BPF LSM is not active"} + if r := c.report(p, nil); r.Status.AppliedGeneration != 0 || r.Status.Tier != "none" { + t.Fatalf("status %+v, want no applied generation with tier none", r.Status) + } +} + +func TestRefusedPrivilegeKeepsTierOfActiveExecBindings(t *testing.T) { + c, _, _, _ := setup(t) + withPrivilege := policy("policy", "ns") + withPrivilege.Spec.Privilege = privilegeSection(api.ActionDeny) + p := parseTypedPolicy(t, withPrivilege) + sections, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, nil, nil) // PrivilegeNotConfigured + c.backend = sections + if r := c.report(p, nil); r.Status.Tier != "none" || r.Status.Conditions[0].Reason != "PrivilegeNotConfigured" { + t.Fatalf("no active binding: %+v", r.Status) + } + b := binding() + c.active[bindingKey(b)] = installed{binding: b, plan: Plan{Policies: []CompiledPolicy{{Identity: PolicyIdentity{UID: p.identity.UID, Generation: 1}, Revision: "r"}}}} + if r := c.report(p, nil); r.Status.Tier != "bpf-lsm" || r.Status.Conditions[0].Reason != "PrivilegeNotConfigured" { + t.Fatalf("last-good exec binding: %+v", r.Status) + } +} + +type podReportingBackend struct { + fakeBackend + pods, faulted []Plan +} + +func (b *podReportingBackend) BoundPodPlans() (confirmed, faulted []Plan) { return b.pods, b.faulted } + +// A pod binding whose slot has a fault cannot be confirmed: it never counts as +// enforcement, and the policy reports that protection is unconfirmed. +func TestFaultedPodBindingIsUnconfirmed(t *testing.T) { + c, _, _, _ := setup(t) + p := parseTypedPolicy(t, policy("policy", "ns")) + p.identity.Generation = 3 + backend := &podReportingBackend{fakeBackend: fakeBackend{plans: map[string]Plan{}}} + backend.faulted = []Plan{{Policies: []CompiledPolicy{{Identity: PolicyIdentity{UID: p.identity.UID, Generation: 3}}}}} + c.backend = backend + r := c.report(p, nil) + if r.Status.AppliedGeneration != 0 || r.Status.Conditions[0].Status != metav1.ConditionFalse || r.Status.Conditions[0].Reason != "ProtectionUnconfirmed" { + t.Fatalf("status %+v", r.Status) + } +} + +// A pod cgroup binding enforces even when the container binding failed, so the +// report must not say that nothing enforces. +func TestPodBindingsCountAsEnforcement(t *testing.T) { + c, _, _, _ := setup(t) + p := parseTypedPolicy(t, policy("policy", "ns")) + p.identity.Generation = 3 + backend := &podReportingBackend{fakeBackend: fakeBackend{plans: map[string]Plan{}}} + backend.pods = []Plan{{Policies: []CompiledPolicy{ + {Identity: PolicyIdentity{UID: p.identity.UID, Generation: 2}, Revision: "r2"}, + {Identity: PolicyIdentity{UID: "other", Generation: 1}, Revision: "o"}, + }}} + c.backend = backend + r := c.report(p, errors.New("container binding refused")) + if r.Status.AppliedGeneration != 2 || r.Status.Tier != "bpf-lsm" || !r.Status.Stale { + t.Fatalf("status %+v", r.Status) + } + if strings.Contains(r.Status.Conditions[0].Message, "no last-good") { + t.Fatalf("pod binding ignored: %q", r.Status.Conditions[0].Message) + } +} + +// Bindings on the desired generation are not stale, even when another binding +// failed; pod plans carry no revision, so ActiveRevision shows none. +func TestPartialFailureOnCurrentGenerationIsNotStale(t *testing.T) { + c, _, _, _ := setup(t) + p := parseTypedPolicy(t, policy("policy", "ns")) + p.identity.Generation = 3 + backend := &podReportingBackend{fakeBackend: fakeBackend{plans: map[string]Plan{}}} + backend.pods = []Plan{{Policies: []CompiledPolicy{{Identity: PolicyIdentity{UID: p.identity.UID, Generation: 3}}}}} + c.backend = backend + r := c.report(p, errors.New("exec binding refused")) + if r.Status.Stale || r.Status.AppliedGeneration != 3 || r.Status.Conditions[0].Status != metav1.ConditionFalse { + t.Fatalf("status %+v", r.Status) + } + if !strings.Contains(r.Status.Conditions[0].Message, "applied in part") { + t.Fatalf("message %q", r.Status.Conditions[0].Message) + } + if active := r.Status.Conditions[1]; active.Type != "ActiveRevision" || active.Message != "generation=3" { + t.Fatalf("active revision %+v", active) + } +} + +// Without BPF LSM the message names the node-wide cause, not the first +// section that also failed for it. +func TestNodeWideUnavailableMessageWins(t *testing.T) { + c, _, _, _ := setup(t) + withPrivilege := policy("policy", "ns") + withPrivilege.Spec.Privilege = privilegeSection(api.ActionDeny) + p := parseTypedPolicy(t, withPrivilege) + c.options.EnforcementUnavailable = &EnforcementUnavailableError{Section: "sandbox policy", Reason: "BPFLSMInactive", Detail: "BPF LSM is not active"} + section := &EnforcementUnavailableError{Section: "privilege", Reason: "BPFLSMInactive", Detail: "BPF LSM is not active"} + r := c.report(p, section) + if ready := r.Status.Conditions[0]; ready.Reason != "BPFLSMInactive" || !strings.HasPrefix(ready.Message, "sandbox policy enforcement") { + t.Fatalf("ready %+v", ready) + } +} + +// On a kernel that is not qualified the privilege section enforces, but a +// privilege policy never reports Ready. +func TestUnqualifiedKernelIsNotReady(t *testing.T) { + c, _, _, _ := setup(t) + withPrivilege := policy("policy", "ns") + withPrivilege.Spec.Privilege = privilegeSection(api.ActionDeny) + p := parseTypedPolicy(t, withPrivilege) + sections, _ := NewSectionBackend(&fakeBackend{plans: map[string]Plan{}}, &fakeBackend{plans: map[string]Plan{}}, nil) + sections.SetPrivilegeUnqualified(&EnforcementUnavailableError{Section: "privilege", Reason: "PrivilegeKernelUnqualified", Detail: "kernel 6.17 is not qualified"}) + c.backend = sections + if r := c.report(p, nil); r.Status.Conditions[0].Status != metav1.ConditionFalse || r.Status.Conditions[0].Reason != "PrivilegeKernelUnqualified" { + t.Fatalf("unqualified kernel: %+v", r.Status.Conditions[0]) + } + if _, err := sections.Replace(context.Background(), Binding{ContainerID: "c"}, sectionPlan("r", CompiledPolicy{Revision: "p", Privilege: privilegeSection(api.ActionDeny)})); err != nil { + t.Fatalf("the privilege section still enforces: %v", err) + } +} + +// Exec failing after the privilege removal leaves a binding that runs neither +// plan: its old revision must not attest. +func TestSectionBackendPartialRemovalDoesNotAttest(t *testing.T) { + exec := &fakeBackend{plans: map[string]Plan{}} + privilege := &fakeBackend{plans: map[string]Plan{}} + s, _ := NewSectionBackend(exec, privilege, nil) + b := Binding{ContainerID: "c"} + execPolicy := CompiledPolicy{Revision: "e", Exec: api.ExecSection{Default: api.ActionDeny}} + privPolicy := CompiledPolicy{Revision: "p", Privilege: privilegeSection(api.ActionAllow)} + if _, err := s.Replace(context.Background(), b, sectionPlan("r1", execPolicy, privPolicy)); err != nil { + t.Fatal(err) + } + exec.fail = true + execPolicy.Revision = "e2" + execPolicy.Exec.Default = api.ActionAllow + if r, err := s.Replace(context.Background(), b, sectionPlan("r2", execPolicy)); err == nil || r.Committed || !r.ProtectionUnconfirmed { + t.Fatalf("exec failure after the removal: %+v %v", r, err) + } + if err := s.Attest(context.Background(), b, "r1"); err == nil { + t.Fatal("the old revision must not attest after a partial removal") + } +} diff --git a/pkg/sandboxpolicy/types.go b/pkg/sandboxpolicy/types.go index dd19d0e40f..aa05404067 100644 --- a/pkg/sandboxpolicy/types.go +++ b/pkg/sandboxpolicy/types.go @@ -45,6 +45,8 @@ type CompiledPolicy struct { Mode api.PolicyMode `json:"mode"` RootRole string `json:"rootRole"` Exec api.ExecSection `json:"exec"` + // Privilege is nil when the policy has no privilege section. + Privilege *api.PrivilegeSection `json:"privilege,omitempty"` } type Plan struct { @@ -101,6 +103,58 @@ type ProtectionAttester interface { Attest(context.Context, Binding, string) error } +// ReplacePreparer optionally does the part of a Replace that can fail without +// changing any binding (for example a new policy slot in a node instance), so +// that a Replace with the same plan right after it is a live map write. +// SectionBackend prepares the privilege section before it commits exec. +type ReplacePreparer interface { + PrepareReplace(ctx context.Context, binding Binding, plan Plan) error +} + +// SlotPreparer optionally prepares enforcement for pods scheduled to this node +// before their containers start. Each plan is what a container of such a pod +// will get. It is a hint: a later Replace must still work without it. +type SlotPreparer interface { + Prepare(context.Context, []Plan) error +} + +// PodBinding places the pod-level cgroup of a pod scheduled to this node in +// the plan that every container of the pod gets. Container cgroups are created +// under the pod cgroup, so a container is judged from its first exec, before +// its own binding exists. The container binding still decides once it exists. +type PodBinding struct { + PodUID types.UID + Namespace string + PodName string + CgroupID uint64 + CgroupPath string + Plan Plan + // ContainersStarted: a container of the pod exists already (the + // container collection knows it). A pod binding does not arm the tasks + // already in the pod cgroup, so such a pod gets no new pod binding, and a + // pod binding carried into a new instance is not confirmed. Its + // containers rely on their own container bindings. + ContainersStarted bool +} + +// PodBinder optionally binds pod cgroups. BindPods receives the complete set: +// a pod binding that is not in it is removed. A pod given with an empty Plan +// keeps its current binding (its input is incomplete). Like SlotPreparer, it is a hint +// that changes no container binding. +type PodBinder interface { + BindPods(context.Context, []PodBinding) error +} + +// PodPlanReporter optionally reports the plans of the pod cgroup bindings, so +// node status counts them like container bindings. confirmed are the plans +// that the witness counters prove: the pod was bound before its containers +// started and its slot has no fault. faulted are the plans whose slot has a +// fault, so their protection cannot be confirmed. A pod bound after its +// containers started is in neither: its containers rely on their own bindings. +type PodPlanReporter interface { + BoundPodPlans() (confirmed, faulted []Plan) +} + // SourceResolver reads ContainerProfile execs in the selected pod's namespace. // It must return an error for missing/incomplete profiles; empty successful results // mean a known complete profile with no execs. Rules retain their predicates.