diff --git a/.github/actions/deploy-prod/action.yml b/.github/actions/deploy-prod/action.yml index f3539c9897..7bcf8bed9e 100644 --- a/.github/actions/deploy-prod/action.yml +++ b/.github/actions/deploy-prod/action.yml @@ -666,6 +666,8 @@ runs: steps.wait_prod_api_stability.outcome == 'success' ) shell: bash + env: + HCLOUD_TOKEN: ${{ inputs.hcloud-token }} run: | go run ./scripts/gateway-receiver-binding restore-disabled --context admin@prod go run ./scripts/gateway-receiver-binding apply-selected --context admin@prod @@ -681,6 +683,8 @@ runs: steps.publish_platform_manifest.outcome == 'success' && steps.wait_flux_revision.outcome != 'success' shell: bash + env: + HCLOUD_TOKEN: ${{ inputs.hcloud-token }} run: go run ./scripts/gateway-receiver-binding restore-disabled --context admin@prod - name: Verify the deployed data product UI release diff --git a/docs/operations/gateway-receiver-boundary.md b/docs/operations/gateway-receiver-boundary.md index e70deba0d1..417796af30 100644 --- a/docs/operations/gateway-receiver-boundary.md +++ b/docs/operations/gateway-receiver-boundary.md @@ -63,6 +63,25 @@ actual Service and Gateway identities. It requires both unbound policy specs to match reviewed source and the validating policy's current generation to have completed type checking without warnings. +Receiver operations also require a read-only observation of the load balancer's +effective disabled transport. The Service annotation and `ipMode` are controller +intent, so they do not establish that the provider has applied the setting. +Using the existing protected deployment credential, the helper discovers the +unique load balancer labelled for the bound Service UID and reads it again by +its returned identifier. Both services must explicitly report PROXY disabled, +with the bound public/private endpoints, destination NodePorts and private server +targets. Missing, partial, changed or unreadable facts fail before a receiver +operation. A second provider observation brackets a successful operation and +must retain its identity. These are bounded observations, not an atomic lock +against independent provider changes. + +The helper sends only GET requests to the fixed provider API origin, refuses +redirects and incomplete inventory, and prints categorical errors. Neither +credentials nor provider response content enter its output. Selected-fence +installation, unselected restoration and deployed public verification use this +same gate. This adds no transport activation or asynchronous recovery support; +an effective active setting still refuses the disabled baseline. + Before restoring disabled state, the helper checks the generated listener and both bindings. An active PROXY listener, unknown owner, foreign binding or ambiguous configuration refuses every planned write. The helper diff --git a/scripts/gateway-receiver-binding/binding.go b/scripts/gateway-receiver-binding/binding.go index 655aba1538..cf88a4d3ac 100644 --- a/scripts/gateway-receiver-binding/binding.go +++ b/scripts/gateway-receiver-binding/binding.go @@ -9,7 +9,10 @@ import ( "regexp" ) -type binding struct{ CIDR, ServiceUID, GatewayUID, ServiceVersion, GatewayVersion, PublicIPv4 string } +type binding struct { + CIDR, ServiceUID, GatewayUID, ServiceVersion, GatewayVersion, PublicIPv4 string + HTTPNodePort, HTTPSNodePort int +} type metadata struct { Name, Namespace, UID, ResourceVersion string Generation int64 @@ -142,12 +145,18 @@ func derive(service, gateway []byte) (binding, error) { } ports := map[int]bool{} nodePorts := map[int]bool{} + var httpNodePort, httpsNodePort int for _, p := range s.Spec.Ports { if (p.Port != 80 && p.Port != 443) || p.Protocol != "TCP" || p.NodePort < 30000 || p.NodePort > 32767 || ports[p.Port] || nodePorts[p.NodePort] { return refuse() } ports[p.Port] = true nodePorts[p.NodePort] = true + if p.Port == 80 { + httpNodePort = p.NodePort + } else { + httpsNodePort = p.NodePort + } } if len(s.Status.LoadBalancer.Ingress) < 2 || len(s.Status.LoadBalancer.Ingress) > 3 { return refuse() @@ -176,5 +185,5 @@ func derive(service, gateway []byte) (binding, error) { if !private.IsValid() || !public.IsValid() { return refuse() } - return binding{CIDR: netip.PrefixFrom(private, 32).String(), ServiceUID: s.Metadata.UID, GatewayUID: g.Metadata.UID, ServiceVersion: s.Metadata.ResourceVersion, GatewayVersion: g.Metadata.ResourceVersion, PublicIPv4: public.String()}, nil + return binding{CIDR: netip.PrefixFrom(private, 32).String(), ServiceUID: s.Metadata.UID, GatewayUID: g.Metadata.UID, ServiceVersion: s.Metadata.ResourceVersion, GatewayVersion: g.Metadata.ResourceVersion, PublicIPv4: public.String(), HTTPNodePort: httpNodePort, HTTPSNodePort: httpsNodePort}, nil } diff --git a/scripts/gateway-receiver-binding/binding_test.go b/scripts/gateway-receiver-binding/binding_test.go index 5721abcda9..b10c0a7d9b 100644 --- a/scripts/gateway-receiver-binding/binding_test.go +++ b/scripts/gateway-receiver-binding/binding_test.go @@ -58,6 +58,17 @@ func TestProviderBindingRequiresOnePrivateHostAndItsGatewayOwner(t *testing.T) { }) } } + +func TestBindingRetainsServiceDestinationsForProviderReadback(t *testing.T) { + service := serviceFixture() + ports := service["spec"].(map[string]any)["ports"].([]any) + ports[0], ports[1] = ports[1], ports[0] + raw, _ := json.Marshal(service) + b, err := derive(raw, gatewayFixture()) + if err != nil || b.HTTPNodePort != 31080 || b.HTTPSNodePort != 31443 { + t.Fatal("provider destinations were lost or associated with list order") + } +} func TestGatewayBindingRefusesUnobservedControllerState(t *testing.T) { raw, _ := json.Marshal(serviceFixture()) var g map[string]any diff --git a/scripts/gateway-receiver-binding/provider.go b/scripts/gateway-receiver-binding/provider.go new file mode 100644 index 0000000000..ef0774afcf --- /dev/null +++ b/scripts/gateway-receiver-binding/provider.go @@ -0,0 +1,204 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/netip" + "net/url" + "os" + "strconv" + "strings" + "time" +) + +type providerIdentity struct{ ID, NetworkID int64 } + +var errProviderMode = errors.New("provider-mode") + +func readProvider(ctx context.Context, b binding, wantProxy bool) (providerIdentity, error) { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + defer transport.CloseIdleConnections() + return observeProvider(ctx, &http.Client{Transport: transport}, os.Getenv("HCLOUD_TOKEN"), b, wantProxy) +} + +type providerBalancer struct { + ID int64 + Name string + Labels map[string]string + PublicNet struct { + Enabled *bool + IPv4 struct { + IP string + Blocked *bool + } + } `json:"public_net"` + PrivateNet []struct { + Network int64 + IP string + } `json:"private_net"` + Services []struct { + Protocol string + ListenPort int `json:"listen_port"` + DestinationPort int `json:"destination_port"` + ProxyProtocol *bool `json:"proxyprotocol"` + } + Targets []struct { + Type string + Server *struct{ ID int64 } + UsePrivateIP *bool `json:"use_private_ip"` + IP any + LabelSelector any `json:"label_selector"` + Targets []json.RawMessage + } +} + +// providerMatches binds effective provider configuration to the already verified +// Kubernetes identity. An absent boolean cannot prove that a mode is disabled. +func providerMatches(lb providerBalancer, b binding, wantProxy bool) (providerIdentity, error) { + bad := func() (providerIdentity, error) { return providerIdentity{}, errors.New("provider-state") } + if lb.ID <= 0 || lb.Name != "platform-gateway" || lb.Labels["hcloud-ccm/service-uid"] != b.ServiceUID || lb.PublicNet.Enabled == nil || !*lb.PublicNet.Enabled || lb.PublicNet.IPv4.IP != b.PublicIPv4 || lb.PublicNet.IPv4.Blocked == nil || *lb.PublicNet.IPv4.Blocked || len(lb.PrivateNet) != 1 || len(lb.Services) != 2 || len(lb.Targets) == 0 { + return bad() + } + private, err := netip.ParsePrefix(b.CIDR) + if err != nil || private.Bits() != 32 || !private.Addr().Is4() || !private.Addr().IsPrivate() || lb.PrivateNet[0].IP != private.Addr().String() || lb.PrivateNet[0].Network <= 0 { + return bad() + } + ports := map[int]bool{} + modeMismatch := false + for _, service := range lb.Services { + destination := b.HTTPNodePort + if service.ListenPort == 443 { + destination = b.HTTPSNodePort + } + if (service.ListenPort != 80 && service.ListenPort != 443) || ports[service.ListenPort] || service.Protocol != "tcp" || service.DestinationPort != destination || service.ProxyProtocol == nil { + return bad() + } + if *service.ProxyProtocol != wantProxy { + modeMismatch = true + } + ports[service.ListenPort] = true + } + servers := map[int64]bool{} + for _, target := range lb.Targets { + if target.Type != "server" || target.Server == nil || target.Server.ID <= 0 || servers[target.Server.ID] || target.UsePrivateIP == nil || !*target.UsePrivateIP || target.IP != nil || target.LabelSelector != nil || len(target.Targets) != 0 { + return bad() + } + servers[target.Server.ID] = true + } + if modeMismatch { + return providerIdentity{}, errProviderMode + } + return providerIdentity{ID: lb.ID, NetworkID: lb.PrivateNet[0].Network}, nil +} + +// Provider schema keys use lowercase ASCII, digits and underscores; labels are +// the sole arbitrary-key map. An allowlist also rejects Unicode simple-fold +// aliases accepted by Go's struct decoder, rather than enumerating exceptions. +// Validate that spelling before Go's case-insensitive struct decoder can allow +// an alias to stand in for, or override, an authoritative mode field. +func providerKeys(v any) bool { + switch value := v.(type) { + case map[string]any: + for key, item := range value { + invalid := strings.IndexFunc(key, func(r rune) bool { + return (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '_' + }) + if key == "" || invalid != -1 || (key != "labels" && !providerKeys(item)) { + return false + } + } + case []any: + for _, item := range value { + if !providerKeys(item) { + return false + } + } + } + return true +} + +func providerJSON(raw []byte, target any) error { + var keys any + if strictJSON(raw, &keys) != nil || !providerKeys(keys) || json.Unmarshal(raw, target) != nil { + return errors.New("provider-document") + } + return nil +} + +// observeProvider performs two bounded GETs to a fixed API origin. Discovery is +// scoped by Service UID, then re-read by the returned resource ID. It makes no +// provider changes and returns only categorical errors, never response bodies. +func observeProvider(ctx context.Context, client *http.Client, token string, b binding, wantProxy bool) (providerIdentity, error) { + if token == "" || strings.ContainsAny(token, " \r\n\t") || client == nil || !uuid.MatchString(b.ServiceUID) || b.HTTPNodePort < 30000 || b.HTTPNodePort > 32767 || b.HTTPSNodePort < 30000 || b.HTTPSNodePort > 32767 || b.HTTPNodePort == b.HTTPSNodePort { + return providerIdentity{}, errors.New("provider-input") + } + bounded := *client + bounded.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + if bounded.Timeout <= 0 || bounded.Timeout > 20*time.Second { + bounded.Timeout = 20 * time.Second + } + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + get := func(path string, query url.Values, target any) error { + u := url.URL{Scheme: "https", Host: "api.hetzner.cloud", Path: path, RawQuery: query.Encode()} + request, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return errors.New("provider-request") + } + request.Header.Set("Authorization", "Bearer "+token) + response, err := bounded.Do(request) + if err != nil { + return errors.New("provider-transport") + } + raw, readErr := io.ReadAll(io.LimitReader(response.Body, (512<<10)+1)) + closeErr := response.Body.Close() + if response.StatusCode != http.StatusOK { + return errors.New("provider-response") + } + if readErr != nil || closeErr != nil || providerJSON(raw, target) != nil { + return errors.New("provider-document") + } + return nil + } + var response struct { + Balancers []providerBalancer `json:"load_balancers"` + Meta struct { + Pagination struct { + Page *int `json:"page"` + PerPage *int `json:"per_page"` + LastPage *int `json:"last_page"` + TotalEntries *int `json:"total_entries"` + NextPage json.RawMessage `json:"next_page"` + } `json:"pagination"` + } `json:"meta"` + } + if err := get("/v1/load_balancers", url.Values{"label_selector": {"hcloud-ccm/service-uid=" + b.ServiceUID}, "page": {"1"}, "per_page": {"50"}}, &response); err != nil { + return providerIdentity{}, err + } + p := response.Meta.Pagination + // A unique filtered result fits one page. Any indication of another page, + // omitted pagination, or a different count is unverified, never truncated. + if p.Page == nil || *p.Page != 1 || p.PerPage == nil || *p.PerPage != 50 || p.LastPage == nil || *p.LastPage != 1 || p.TotalEntries == nil || *p.TotalEntries != 1 || !bytes.Equal(bytes.TrimSpace(p.NextPage), []byte("null")) || len(response.Balancers) != 1 { + return providerIdentity{}, errors.New("provider-inventory") + } + before, err := providerMatches(response.Balancers[0], b, wantProxy) + if err != nil { + return providerIdentity{}, err + } + var detail struct { + Balancer providerBalancer `json:"load_balancer"` + } + if err = get("/v1/load_balancers/"+strconv.FormatInt(before.ID, 10), nil, &detail); err != nil { + return providerIdentity{}, err + } + after, err := providerMatches(detail.Balancer, b, wantProxy) + if err != nil || before != after { + return providerIdentity{}, errors.New("provider-changed") + } + return after, nil +} diff --git a/scripts/gateway-receiver-binding/provider_guard.go b/scripts/gateway-receiver-binding/provider_guard.go new file mode 100644 index 0000000000..9ed46e0db3 --- /dev/null +++ b/scripts/gateway-receiver-binding/provider_guard.go @@ -0,0 +1,29 @@ +package main + +import ( + "context" + "errors" +) + +// A Service annotation expresses desired state. Gate receiver operations on the +// provider's effective disabled state and retain the same provider identity. +func withProviderBaseline(ctx context.Context, b binding, observe func(context.Context, binding, bool) (providerIdentity, error), operation func() error) error { + if observe == nil || operation == nil { + return errors.New("provider-capability") + } + before, err := observe(ctx, b, false) + if err != nil { + return err + } + if err = operation(); err != nil { + return err + } + after, err := observe(ctx, b, false) + if err != nil { + return err + } + if after != before { + return errors.New("provider-changed") + } + return nil +} diff --git a/scripts/gateway-receiver-binding/provider_guard_test.go b/scripts/gateway-receiver-binding/provider_guard_test.go new file mode 100644 index 0000000000..50e9c45aa5 --- /dev/null +++ b/scripts/gateway-receiver-binding/provider_guard_test.go @@ -0,0 +1,56 @@ +package main + +import ( + "context" + "errors" + "testing" +) + +func TestProviderBaselineGuardsTheOperationAndPinsTheObservation(t *testing.T) { + for _, scenario := range []string{"disabled", "missing-capability", "unknown-before", "active-before", "changed-after", "unknown-after", "operation-failed"} { + t.Run(scenario, func(t *testing.T) { + _, b := providerFixture() + reads, operations := 0, 0 + reader := func(_ context.Context, actual binding, wantProxy bool) (providerIdentity, error) { + reads++ + if actual != b || wantProxy { + t.Fatal("operation was not bound to the disabled provider profile") + } + if (scenario == "unknown-before" && reads == 1) || (scenario == "unknown-after" && reads == 2) { + return providerIdentity{}, errors.New("provider-document") + } + if scenario == "active-before" && reads == 1 { + return providerIdentity{}, errProviderMode + } + id := int64(7) + if scenario == "changed-after" && reads == 2 { + id = 8 + } + return providerIdentity{ID: id, NetworkID: 9}, nil + } + if scenario == "missing-capability" { + reader = nil + } + err := withProviderBaseline(context.Background(), b, reader, func() error { + operations++ + if scenario == "operation-failed" { + return errors.New("source-admission-policy") + } + return nil + }) + wantOperations := 1 + if scenario == "missing-capability" || scenario == "unknown-before" || scenario == "active-before" { + wantOperations = 0 + } + if operations != wantOperations || (err == nil) != (scenario == "disabled") { + t.Fatalf("provider disagreement or missing proof permitted an operation or success: operations=%d error=%v", operations, err) + } + if scenario == "disabled" && reads != 2 { + t.Fatal("operation lacked provider readback on both sides") + } + if scenario == "operation-failed" && (err == nil || err.Error() != "source-admission-policy") { + t.Fatal("a successful provider observation hid the operation failure") + } + }) + } +} diff --git a/scripts/gateway-receiver-binding/provider_test.go b/scripts/gateway-receiver-binding/provider_test.go new file mode 100644 index 0000000000..ffceb452a3 --- /dev/null +++ b/scripts/gateway-receiver-binding/provider_test.go @@ -0,0 +1,310 @@ +package main + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" +) + +// Fixtures use documentation addresses and a disposable bearer value only. +func providerFixture() (map[string]any, binding) { + s, _ := json.Marshal(serviceFixture()) + b, _ := derive(s, gatewayFixture()) + lb := map[string]any{ + "id": 7, "name": "platform-gateway", + "labels": map[string]any{"hcloud-ccm/service-uid": b.ServiceUID}, + "public_net": map[string]any{"enabled": true, "ipv4": map[string]any{"ip": b.PublicIPv4, "blocked": false}}, + "private_net": []any{map[string]any{"network": 9, "ip": "10.11.12.13"}}, + "services": []any{ + map[string]any{"listen_port": 80, "destination_port": 31080, "protocol": "tcp", "proxyprotocol": false}, + map[string]any{"listen_port": 443, "destination_port": 31443, "protocol": "tcp", "proxyprotocol": false}, + }, + "targets": []any{map[string]any{"type": "server", "server": map[string]any{"id": 11}, "use_private_ip": true}}, + } + return lb, b +} + +type providerTransport func(*http.Request) (*http.Response, error) + +func (f providerTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func providerResponse(v any) *http.Response { + raw, _ := json.Marshal(v) + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(string(raw))), Header: make(http.Header)} +} + +func providerHTTP(t *testing.T, lb map[string]any) (*http.Client, *int) { + t.Helper() + calls := new(int) + client := &http.Client{Transport: providerTransport(func(r *http.Request) (*http.Response, error) { + *calls++ + if r.Method != http.MethodGet || r.URL.Scheme != "https" || r.URL.Host != "api.hetzner.cloud" || r.Header.Get("Authorization") != "Bearer disposable-test-token" || r.Body != nil { + t.Fatal("provider observation changed method, origin or credential boundary") + } + switch r.URL.Path { + case "/v1/load_balancers": + if r.URL.Query().Get("label_selector") != "hcloud-ccm/service-uid=22222222-2222-4222-8222-222222222222" || r.URL.Query().Get("page") != "1" || r.URL.Query().Get("per_page") != "50" { + t.Fatal("provider lookup was not scoped to the bound Service UID") + } + return providerResponse(map[string]any{"load_balancers": []any{lb}, "meta": map[string]any{"pagination": map[string]any{"page": 1, "per_page": 50, "last_page": 1, "next_page": nil, "total_entries": 1}}}), nil + case "/v1/load_balancers/7": + return providerResponse(map[string]any{"load_balancer": lb}), nil + default: + t.Fatal("provider observation requested an undeclared resource") + return nil, nil + } + })} + return client, calls +} + +func TestProviderObservationRequiresEffectiveModeAndBoundIdentity(t *testing.T) { + lb, b := providerFixture() + client, calls := providerHTTP(t, lb) + got, err := observeProvider(context.Background(), client, "disposable-test-token", b, false) + if err != nil || got != (providerIdentity{ID: 7, NetworkID: 9}) || *calls != 2 { + t.Fatalf("complete read-only provider observation refused: %v", err) + } + for _, item := range lb["services"].([]any) { + item.(map[string]any)["proxyprotocol"] = true + } + got, err = observeProvider(context.Background(), client, "disposable-test-token", b, true) + if err != nil || got.ID != 7 { + t.Fatal("known active state cannot be inspected for recovery") + } +} + +func TestProviderObservationRefusesMissingOrConflictingFacts(t *testing.T) { + for _, bad := range []string{"uid", "name", "id", "public", "public-disabled", "public-blocked", "private", "network", "extra-network", "missing-mode", "null-mode", "enabled", "partial-mode", "destination", "protocol", "extra-service", "duplicate-service", "target-public", "target-ip", "empty-targets"} { + t.Run(bad, func(t *testing.T) { + lb, b := providerFixture() + svc := lb["services"].([]any)[1].(map[string]any) + switch bad { + case "uid": + lb["labels"].(map[string]any)["hcloud-ccm/service-uid"] = "other" + case "name": + lb["name"] = "other" + case "id": + lb["id"] = 0 + case "public": + lb["public_net"].(map[string]any)["ipv4"].(map[string]any)["ip"] = "192.0.2.2" + case "public-disabled": + lb["public_net"].(map[string]any)["enabled"] = false + case "public-blocked": + lb["public_net"].(map[string]any)["ipv4"].(map[string]any)["blocked"] = true + case "private": + lb["private_net"].([]any)[0].(map[string]any)["ip"] = "10.11.12.14" + case "network": + lb["private_net"].([]any)[0].(map[string]any)["network"] = 0 + case "extra-network": + lb["private_net"] = append(lb["private_net"].([]any), map[string]any{"network": 10, "ip": "10.11.12.14"}) + case "missing-mode": + delete(svc, "proxyprotocol") + case "null-mode": + svc["proxyprotocol"] = nil + case "enabled": + for _, item := range lb["services"].([]any) { + item.(map[string]any)["proxyprotocol"] = true + } + case "partial-mode": + svc["proxyprotocol"] = true + case "destination": + svc["destination_port"] = 443 + case "protocol": + svc["protocol"] = "https" + case "extra-service": + lb["services"] = append(lb["services"].([]any), map[string]any{"listen_port": 25}) + case "duplicate-service": + svc["listen_port"] = 80 + case "target-public": + lb["targets"].([]any)[0].(map[string]any)["use_private_ip"] = false + case "target-ip": + lb["targets"].([]any)[0].(map[string]any)["type"] = "ip" + case "empty-targets": + lb["targets"] = []any{} + } + client, _ := providerHTTP(t, lb) + if _, err := observeProvider(context.Background(), client, "disposable-test-token", b, false); err == nil { + t.Fatal("unverified provider state accepted") + } + }) + } +} + +func TestProviderObservationRefusesIncompleteInventoryAndChangedDetail(t *testing.T) { + for _, bad := range []string{"missing-meta", "no-match", "multiple", "more-pages", "missing-next", "unknown-count", "wrong-page", "duplicate-json", "truncated-json", "detail-id", "detail-network", "detail-mode", "detail-uid"} { + t.Run(bad, func(t *testing.T) { + lb, b := providerFixture() + client, _ := providerHTTP(t, lb) + base := client.Transport + client.Transport = providerTransport(func(r *http.Request) (*http.Response, error) { + resp, err := base.RoundTrip(r) + if err != nil { + return resp, err + } + defer func() { _ = resp.Body.Close() }() + var doc map[string]any + if json.NewDecoder(resp.Body).Decode(&doc) != nil { + t.Fatal("fixture response") + } + if r.URL.Path == "/v1/load_balancers" { + p := doc["meta"].(map[string]any)["pagination"].(map[string]any) + switch bad { + case "missing-meta": + delete(doc, "meta") + case "no-match": + doc["load_balancers"] = []any{} + case "multiple": + doc["load_balancers"] = []any{lb, lb} + p["total_entries"] = 2 + case "more-pages": + p["next_page"] = 2 + p["last_page"] = 2 + case "missing-next": + delete(p, "next_page") + case "unknown-count": + delete(p, "total_entries") + case "wrong-page": + p["page"] = 2 + case "duplicate-json": + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(`{"load_balancers":[],"load_balancers":[]}`))}, nil + case "truncated-json": + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(`{"load_balancers": [`))}, nil + } + } else { + detail := doc["load_balancer"].(map[string]any) + switch bad { + case "detail-id": + detail["id"] = 8 + case "detail-network": + detail["private_net"].([]any)[0].(map[string]any)["network"] = 10 + case "detail-mode": + detail["services"].([]any)[1].(map[string]any)["proxyprotocol"] = true + case "detail-uid": + detail["labels"].(map[string]any)["hcloud-ccm/service-uid"] = "other" + } + } + return providerResponse(doc), nil + }) + if _, err := observeProvider(context.Background(), client, "disposable-test-token", b, false); err == nil { + t.Fatal("incomplete or changed provider evidence accepted") + } + }) + } +} + +func TestProviderObservationRequiresCanonicalJSONFields(t *testing.T) { + for _, where := range []string{"list", "detail"} { + for _, duplicate := range []bool{false, true} { + t.Run(where+"/"+map[bool]string{false: "alias", true: "collision"}[duplicate], func(t *testing.T) { + lb, b := providerFixture() + client, _ := providerHTTP(t, lb) + base := client.Transport + client.Transport = providerTransport(func(r *http.Request) (*http.Response, error) { + resp, err := base.RoundTrip(r) + if err != nil { + return resp, err + } + if (where == "list") != (r.URL.Path == "/v1/load_balancers") { + return resp, nil + } + data, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + field := `"ProxyProtocol":false` + if duplicate { + field = `"proxyprotocol":true,"ProxyProtocol":false` + } + data = []byte(strings.Replace(string(data), `"proxyprotocol":false`, field, 1)) + resp.Body = io.NopCloser(strings.NewReader(string(data))) + return resp, nil + }) + if _, err := observeProvider(context.Background(), client, "disposable-test-token", b, false); err == nil { + t.Fatal("case-insensitive decoder accepted a missing or overridden authoritative field") + } + }) + } + } +} + +func TestProviderObservationAcceptsEmptyServerChildrenAndArbitraryLabelCase(t *testing.T) { + lb, b := providerFixture() + lb["targets"].([]any)[0].(map[string]any)["targets"] = []any{} + lb["labels"].(map[string]any)["BusinessOwner"] = "synthetic" + client, _ := providerHTTP(t, lb) + if _, err := observeProvider(context.Background(), client, "disposable-test-token", b, false); err != nil { + t.Fatal("valid provider server leaf or arbitrary label refused") + } +} + +func TestProviderObservationRefusesUnicodeFoldAliases(t *testing.T) { + for _, where := range []string{"list", "detail"} { + for _, duplicate := range []bool{false, true} { + t.Run(where+"/"+map[bool]string{false: "alias", true: "collision"}[duplicate], func(t *testing.T) { + lb, b := providerFixture() + client, _ := providerHTTP(t, lb) + base := client.Transport + client.Transport = providerTransport(func(r *http.Request) (*http.Response, error) { + resp, err := base.RoundTrip(r) + if err != nil { + return resp, err + } + if (where == "list") != (r.URL.Path == "/v1/load_balancers") { + return resp, nil + } + data, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + field := `"\u017fervices":` + if duplicate { + field = `"services":[{"listen_port":80,"destination_port":31080,"protocol":"tcp","proxyprotocol":true},{"listen_port":443,"destination_port":31443,"protocol":"tcp","proxyprotocol":true}],` + field + } + data = []byte(strings.Replace(string(data), `"services":`, field, 1)) + resp.Body = io.NopCloser(strings.NewReader(string(data))) + return resp, nil + }) + if _, err := observeProvider(context.Background(), client, "disposable-test-token", b, false); err == nil { + t.Fatal("Unicode-folded alias substituted an authoritative schema field") + } + }) + } + } +} + +func TestProviderObservationDoesNotFollowRedirectOrDiscloseProviderBody(t *testing.T) { + forwarded := 0 + other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { forwarded++; w.WriteHeader(200) })) + defer other.Close() + _, b := providerFixture() + for _, status := range []int{301, 302, 307, 401, 403, 429, 500} { + t.Run(http.StatusText(status), func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" || r.Header.Get("Authorization") != "Bearer disposable-test-token" { + t.Error("request boundary") + } + w.Header().Set("Location", other.URL) + w.WriteHeader(status) + _, _ = io.WriteString(w, "private-provider-detail disposable-test-token") + })) + defer server.Close() + local, _ := url.Parse(server.URL) + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + defer transport.CloseIdleConnections() + client := &http.Client{Transport: providerTransport(func(r *http.Request) (*http.Response, error) { + if r.URL.Host != "api.hetzner.cloud" { + t.Fatal("redirect followed") + } + copy := r.Clone(r.Context()) + copy.URL.Scheme, copy.URL.Host = local.Scheme, local.Host + return transport.RoundTrip(copy) + })} + _, err := observeProvider(context.Background(), client, "disposable-test-token", b, false) + if err == nil || strings.Contains(err.Error(), "private-provider-detail") || strings.Contains(err.Error(), "disposable-test-token") || forwarded != 0 { + t.Fatal("HTTP refusal leaked data or followed a redirect") + } + }) + } +} diff --git a/scripts/gateway-receiver-binding/registry.go b/scripts/gateway-receiver-binding/registry.go index c1e52111b2..fcc2a84f2a 100644 --- a/scripts/gateway-receiver-binding/registry.go +++ b/scripts/gateway-receiver-binding/registry.go @@ -235,24 +235,26 @@ func restoreDisabled(ctx context.Context, kubeContext string) error { if err != nil { return err } - c := receiverClient{context: kubeContext, command: executeReceiver} - if err = c.restoreReceiver(ctx, before); err != nil && !isPolicyDrift(err) { - return err - } - restored := err - // A stale binding is reported after the removal for the same reason. - secret, err := readNamespace(ctx, kubeContext, "flux-system", "secret", "variables-gateway-receiver") - if err != nil || !secretMatches(secret, before) { - return errors.New("source-secret") - } - if restored != nil { - return restored - } - after, err := collect(ctx, kubeContext) - if err != nil || before != after { - return errors.New("source-changed") - } - return c.bindingsAbsent(ctx) + return withProviderBaseline(ctx, before, readProvider, func() error { + c := receiverClient{context: kubeContext, command: executeReceiver} + if err = c.restoreReceiver(ctx, before); err != nil && !isPolicyDrift(err) { + return err + } + restored := err + // A stale binding is reported after the removal for the same reason. + secret, err := readNamespace(ctx, kubeContext, "flux-system", "secret", "variables-gateway-receiver") + if err != nil || !secretMatches(secret, before) { + return errors.New("source-secret") + } + if restored != nil { + return restored + } + after, err := collect(ctx, kubeContext) + if err != nil || before != after { + return errors.New("source-changed") + } + return c.bindingsAbsent(ctx) + }) } func isPolicyDrift(err error) bool { diff --git a/scripts/gateway-receiver-binding/selected.go b/scripts/gateway-receiver-binding/selected.go index 0d1e5a6b32..83a092d466 100644 --- a/scripts/gateway-receiver-binding/selected.go +++ b/scripts/gateway-receiver-binding/selected.go @@ -285,21 +285,26 @@ func applySelected(ctx context.Context, kubeContext string) (bool, error) { } } c := receiverClient{context: kubeContext, command: executeReceiver} - wrote, err := c.installFence(ctx, before, pauseFor) - if err != nil { - return wrote, err - } - // Enough fresh connections to reach most nodes behind the load balancer, once, - // when the listeners change. This shows the fence does not refuse the load - // balancer; it cannot show that a listener took the fence. - for round := 0; wrote && round < fenceProbeRounds; round++ { - if err := probePublic(ctx, before.PublicIPv4); err != nil { - return wrote, err + wrote := false + err = withProviderBaseline(ctx, before, readProvider, func() error { + var installErr error + wrote, installErr = c.installFence(ctx, before, pauseFor) + if installErr != nil { + return installErr } - } - after, err := collect(ctx, kubeContext) - if err != nil || !sameReceiver(before, after) { - return wrote, errors.New("source-changed") - } - return wrote, nil + // Enough fresh connections to reach most nodes behind the load balancer, once, + // when the listeners change. This shows the fence does not refuse the load + // balancer; it cannot show that a listener took the fence. + for round := 0; wrote && round < fenceProbeRounds; round++ { + if err := probePublic(ctx, before.PublicIPv4); err != nil { + return err + } + } + after, err := collect(ctx, kubeContext) + if err != nil || !sameReceiver(before, after) { + return errors.New("source-changed") + } + return nil + }) + return wrote, err } diff --git a/scripts/gateway-receiver-binding/verify.go b/scripts/gateway-receiver-binding/verify.go index 750ef13116..2b149be907 100644 --- a/scripts/gateway-receiver-binding/verify.go +++ b/scripts/gateway-receiver-binding/verify.go @@ -54,32 +54,34 @@ func verifyDeployed(ctx context.Context, kubeContext string, selected bool) erro if err != nil { return err } - for sample := 0; sample < 2; sample++ { - secret, err := readNamespace(ctx, kubeContext, "flux-system", "secret", "variables-gateway-receiver") - if err != nil || !secretMatches(secret, before) { - return errors.New("source-secret") - } - c := receiverClient{context: kubeContext, command: executeReceiver} - if err := c.policies(ctx, before); err != nil { - return err - } - if err := c.receiverState(ctx, before, selected); err != nil { - return err - } - if err = probePublic(ctx, before.PublicIPv4); err != nil { - return err - } - if sample == 0 { - select { - case <-ctx.Done(): - return errors.New("cancelled") - case <-time.After(time.Second): + return withProviderBaseline(ctx, before, readProvider, func() error { + for sample := 0; sample < 2; sample++ { + secret, err := readNamespace(ctx, kubeContext, "flux-system", "secret", "variables-gateway-receiver") + if err != nil || !secretMatches(secret, before) { + return errors.New("source-secret") + } + c := receiverClient{context: kubeContext, command: executeReceiver} + if err := c.policies(ctx, before); err != nil { + return err + } + if err := c.receiverState(ctx, before, selected); err != nil { + return err + } + if err = probePublic(ctx, before.PublicIPv4); err != nil { + return err + } + if sample == 0 { + select { + case <-ctx.Done(): + return errors.New("cancelled") + case <-time.After(time.Second): + } } } - } - after, err := collect(ctx, kubeContext) - if err != nil || after != before { - return errors.New("source-changed") - } - return nil + after, err := collect(ctx, kubeContext) + if err != nil || after != before { + return errors.New("source-changed") + } + return nil + }) }