Skip to content
4 changes: 3 additions & 1 deletion .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -421,6 +421,7 @@ jobs:
# up, and that no address reaches the public log.
- 'scripts/probe-gateway-denied-peer.sh'
- 'scripts/tests/test-probe-gateway-denied-peer.sh'
- 'scripts/tests/gateway-denied-peer-wire/**'
- '.github/workflows/probe-gateway-denied-peer.yaml'
# The #3144 admission probe, its test and its workflow are one
# contract: the test pins that every write the probe sends is a
Expand Down Expand Up @@ -2535,7 +2536,8 @@ jobs:
- name: 🚧 Validate Gateway denied-peer probe
if: needs.changes.outputs.k8s == 'true'
# The probe needs prod, runs only on dispatch and writes while it runs. Its
# test fakes kubectl and pins each verdict against a one-fixture control.
# test pins verdicts and cleanup, then executes both rendered transport modes with
# real curl and synthetic loopback HTTP/TLS peers.
run: |
shellcheck scripts/probe-gateway-denied-peer.sh scripts/tests/test-probe-gateway-denied-peer.sh
bash scripts/tests/test-probe-gateway-denied-peer.sh
Expand Down
16 changes: 13 additions & 3 deletions .github/workflows/probe-gateway-denied-peer.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ jobs:

# The allowed half: the fence is in place and the load balancer's traffic is still answered.
- name: Verify the selected fence and the public routes
id: binding
shell: bash
env:
HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
Expand All @@ -85,19 +86,28 @@ jobs:
verified="$(go run ./scripts/gateway-receiver-binding verify --context admin@prod)"
printf '%s\n' "${verified}"
# Without a selected fence, an answered connection would say nothing about the fence.
grep -Eq '(^| )fence_selected=true( |$)' <<<"${verified}" || { printf 'The fence is not selected; nothing was probed.\n' >&2; exit 1; }
pattern='^GATEWAY_RECEIVER_SOURCE=VERIFIED public_probes=4 fence_selected=true proxy_protocol=(true|false)$'
[[ "${verified}" =~ ${pattern} ]] || { printf 'The fenced transport profile is unverified; nothing was probed.\n' >&2; exit 1; }
transport=plain
[[ "${BASH_REMATCH[1]}" == false ]] || transport=proxy-v1
printf 'transport=%s\n' "${transport}" >>"${GITHUB_OUTPUT}"

- name: Probe with a pod as the peer
id: probe
shell: bash
env:
RUN_ID: ${{ github.run_id }}
run: ./scripts/probe-gateway-denied-peer.sh --context admin@prod --run-id "${RUN_ID}"
TRANSPORT: ${{ steps.binding.outputs.transport }}
run: ./scripts/probe-gateway-denied-peer.sh --context admin@prod --run-id "${RUN_ID}" --transport "${TRANSPORT}"

# A cancelled job can end the step before the script's own cleanup runs.
- name: Remove anything this run left behind
if: ${{ always() && steps.probe.outcome != 'skipped' }}
shell: bash
env:
RUN_ID: ${{ github.run_id }}
run: kubectl --context admin@prod --request-timeout=30s --namespace whoami delete pods,ciliumnetworkpolicies --selector "platform.devantler.tech/gateway-peer-probe=${RUN_ID}" --ignore-not-found >/dev/null 2>&1
run: |
set -euo pipefail
kubectl --context admin@prod --request-timeout=30s --namespace whoami delete pods,ciliumnetworkpolicies --selector "platform.devantler.tech/gateway-peer-probe=${RUN_ID}" --ignore-not-found --wait=true --timeout=25s >/dev/null 2>&1
remaining="$(kubectl --context admin@prod --request-timeout=30s --namespace whoami get pods,ciliumnetworkpolicies --selector "platform.devantler.tech/gateway-peer-probe=${RUN_ID}" -o name 2>/dev/null)"
[[ -z "${remaining}" ]] || { printf 'Run-owned cleanup is unverified.\n' >&2; exit 1; }
2 changes: 1 addition & 1 deletion scripts/gateway-receiver-binding/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ func main() {
fmt.Fprintf(os.Stderr, "GATEWAY_RECEIVER_SOURCE=UNVERIFIED stage=%s\n", err.Error())
os.Exit(2)
}
fmt.Printf("GATEWAY_RECEIVER_SOURCE=VERIFIED public_probes=4 fence_selected=%t\n", profile.Fence)
fmt.Printf("GATEWAY_RECEIVER_SOURCE=VERIFIED public_probes=4 fence_selected=%t proxy_protocol=%t\n", profile.Fence, profile.Proxy)
return
}
if err := prepare(ctx, kubeContext, outputPath); err != nil {
Expand Down
29 changes: 23 additions & 6 deletions scripts/probe-gateway-denied-peer.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,16 @@
#
# THE QUESTION (devantler-tech/platform#5448, stage 3). Every TLS chain on the Gateway's listener
# accepts only the load balancer's private address. So far only allowed traffic has been observed,
# which cannot show the fence refuses anything. This must be answered before any PROXY change,
# which cannot show the fence refuses anything. Both listener transport modes must be testable,
# because PROXY metadata names a client the listener must not take from an unbound peer.
#
# HOW. One short-lived pod connects straight to the gateway Service, so its physical peer is the
# pod and not the load balancer. A pod's connection is handled by the Envoy on its own node.
# control plain HTTP to an unrouted hostname. It must get the platform's HTTP-to-HTTPS redirect
# (301) before and after the TLS attempts, which proves this pod reaches this Envoy and
# that the plain chain, which carries no fence, still answers.
# In proxy-v1 mode every request sends a valid PROXY preamble before HTTP or TLS;
# otherwise the parser's rejection could be mistaken for the physical-source fence.
# tls HTTPS with the SNI of a public route. Certificate checks are off on purpose: a finished
# handshake is the failure this looks for, whatever certificate came back.
# refused curl 35 or 56 with the TLS handshake never finished (time_appconnect is 0)
Expand Down Expand Up @@ -68,15 +70,16 @@ readonly settle_seconds=20
readonly poll_seconds="${PROBE_POLL_SECONDS:-2}"

usage() {
printf 'Usage: %s --context <kube-context> --run-id <digits> [--attempts N]\n' "$(basename "$0")" >&2
printf 'Usage: %s --context <kube-context> --run-id <digits> --transport <plain|proxy-v1> [--attempts N]\n' "$(basename "$0")" >&2
}

context=''
run_id=''
attempts='10'
transport=''
while [[ "$#" -gt 0 ]]; do
case "$1" in
--context | --run-id | --attempts)
--context | --run-id | --attempts | --transport)
if [[ "$#" -lt 2 || -z "$2" ]]; then
usage
exit 1
Expand All @@ -85,6 +88,7 @@ while [[ "$#" -gt 0 ]]; do
--context) context="$2" ;;
--run-id) run_id="$2" ;;
--attempts) attempts="$2" ;;
--transport) transport="$2" ;;
esac
shift 2
;;
Expand All @@ -94,14 +98,19 @@ while [[ "$#" -gt 0 ]]; do
;;
esac
done
if [[ -z "${context}" ]] || [[ ! "${run_id}" =~ ^[1-9][0-9]{0,19}$ ]]; then
if [[ -z "${context}" ]] || [[ ! "${run_id}" =~ ^[1-9][0-9]{0,19}$ ]] || [[ "${transport}" != plain && "${transport}" != proxy-v1 ]]; then
usage
exit 1
fi
if [[ ! "${attempts}" =~ ^[1-9][0-9]?$ ]] || ((attempts > 30)); then
printf 'Refusing to run: --attempts must be a whole number from 1 to 30.\n' >&2
exit 1
fi
curl_transport_option=''
if [[ "${transport}" == proxy-v1 ]]; then
curl_transport_option='--haproxy-protocol'
fi
readonly curl_transport_option

# Worst case: warm-up, the settle wait, every attempt timing out, the closing control, and pull.
readonly pod_deadline_seconds=$(((warmup_attempts + attempts + 1) * attempt_timeout + warmup_attempts + settle_seconds + 120))
Expand Down Expand Up @@ -147,13 +156,21 @@ fi
# shellcheck disable=SC2317,SC2329
cleanup() {
local rc=$?
local remaining
trap - EXIT INT TERM
if ! kc -n "${probe_namespace}" delete pods,ciliumnetworkpolicies -l "${probe_label}=${run_id}" --ignore-not-found --wait=true --timeout=25s >/dev/null; then
printf 'CLEANUP: FAILED, delete objects labelled %s=%s by hand\n' "${probe_label}" "${run_id}"
summary ""
summary "**Cleanup failed:** delete objects labelled \`${probe_label}=${run_id}\` by hand."
exit 4
fi
remaining="$(kc -n "${probe_namespace}" get pods,ciliumnetworkpolicies -l "${probe_label}=${run_id}" -o name)" || remaining='unknown'
if [[ -n "${remaining}" ]]; then
printf 'CLEANUP: FAILED, absence of run-owned objects was not verified\n'
summary ""
summary "**Cleanup failed:** absence of run-owned objects was not verified."
exit 4
fi
printf 'CLEANUP: done\n'
exit "${rc}"
}
Expand Down Expand Up @@ -239,7 +256,7 @@ spec:
- |
set -u
control() {
curl -s -o /dev/null --max-time "\$TIMEOUT" -H "Host: \$CONTROL_HOST" -w '%{http_code}' "http://\$GATEWAY_IP/" 2>/dev/null || true
curl -s ${curl_transport_option} -o /dev/null --max-time "\$TIMEOUT" -H "Host: \$CONTROL_HOST" -w '%{http_code}' "http://\$GATEWAY_IP/" 2>/dev/null || true
}
i=0
first=''
Expand All @@ -255,7 +272,7 @@ spec:
i=0
while [ "\$i" -lt "\$ATTEMPTS" ]; do
i=\$((i + 1))
code="\$(curl -sk -o /dev/null --max-time "\$TIMEOUT" --resolve "\$TLS_HOST:443:\$GATEWAY_IP" -w '%{http_code} %{time_appconnect}' "https://\$TLS_HOST/" 2>/dev/null)"
code="\$(curl -sk ${curl_transport_option} -o /dev/null --max-time "\$TIMEOUT" --resolve "\$TLS_HOST:443:\$GATEWAY_IP" -w '%{http_code} %{time_appconnect}' "https://\$TLS_HOST/" 2>/dev/null)"
printf 'PROBE tls %s %s\\n' "\$?" "\$code"
done
printf 'PROBE control-after %s\\n' "\$(control)"
Expand Down
133 changes: 133 additions & 0 deletions scripts/tests/gateway-denied-peer-wire/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
// Exercise the rendered diagnostic with real curl and synthetic loopback peers.
package main

import (
"bufio"
"bytes"
"context"
"fmt"
"io"
"net"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)

func serve(listener net.Listener, proxy, secure bool, results chan<- error) {
for {
conn, err := listener.Accept()
if err != nil {
return
}
func() {
defer func() { _ = conn.Close() }()
_ = conn.SetDeadline(time.Now().Add(2 * time.Second))
reader := bufio.NewReader(conn)
if proxy {
header, err := reader.ReadString('\n')
if err != nil || !strings.HasPrefix(header, "PROXY TCP4 ") || !strings.HasSuffix(header, "\r\n") || len(strings.Fields(header)) != 6 {
results <- fmt.Errorf("missing valid PROXY preamble")
return
}
}
if secure {
header := make([]byte, 5)
if _, err := io.ReadFull(reader, header); err != nil || header[0] != 22 || header[1] != 3 {
results <- fmt.Errorf("missing TLS ClientHello after transport framing")
return
}
// Deliberately refuse before the handshake, just as a denied physical peer is closed.
results <- nil
return
}
line, err := reader.ReadString('\n')
if err != nil || line != "GET / HTTP/1.1\r\n" {
results <- fmt.Errorf("missing HTTP request after transport framing")
return
}
_, err = io.WriteString(conn, "HTTP/1.1 301 Moved Permanently\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
results <- err
}()
}
}

func run() error {
if len(os.Args) != 3 || os.Args[2] != "plain" && os.Args[2] != "proxy-v1" {
return fmt.Errorf("arguments")
}
httpPeer, err := net.Listen("tcp4", "127.0.0.1:0")
if err != nil {
return err
}
defer func() { _ = httpPeer.Close() }()
tlsPeer, err := net.Listen("tcp4", "127.0.0.1:0")
if err != nil {
return err
}
defer func() { _ = tlsPeer.Close() }()
results := make(chan error, 32)
go serve(httpPeer, os.Args[2] == "proxy-v1", false, results)
go serve(tlsPeer, os.Args[2] == "proxy-v1", true, results)
curl, err := exec.LookPath("curl")
if err != nil {
return err
}
work, err := os.MkdirTemp("", "gateway-denied-wire-")
if err != nil {
return err
}
defer func() { _ = os.RemoveAll(work) }()
wrapper := "#!/bin/sh\nexec \"$WIRE_CURL\" --noproxy '*' --connect-to \"$GATEWAY_IP:80:$WIRE_HTTP\" --connect-to \"$TLS_HOST:443:$WIRE_TLS\" \"$@\"\n"
if err := os.WriteFile(filepath.Join(work, "curl"), []byte(wrapper), 0700); err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
command := exec.CommandContext(ctx, "sh", os.Args[1])
command.Env = append(os.Environ(), "PATH="+work+":"+os.Getenv("PATH"), "WIRE_CURL="+curl,
"WIRE_HTTP="+httpPeer.Addr().String(), "WIRE_TLS="+tlsPeer.Addr().String(),
"ATTEMPTS=2", "WARMUP_ATTEMPTS=1", "TIMEOUT=1", "SETTLE_SECONDS=0",
"GATEWAY_IP=192.0.2.1", "TLS_HOST=example.test", "CONTROL_HOST=x.invalid")
output, err := command.CombinedOutput()
if err != nil || !bytes.Contains(output, []byte("PROBE control-before 301\n")) || !bytes.Contains(output, []byte("PROBE control-after 301\n")) {
return fmt.Errorf("native controls failed: %v: %s", err, output)
}
attempts := 0
for _, line := range strings.Split(strings.TrimSpace(string(output)), "\n") {
if strings.HasPrefix(line, "PROBE tls ") {
attempts++
if !strings.HasPrefix(line, "PROBE tls 35 000 0.000000") && !strings.HasPrefix(line, "PROBE tls 56 000 0.000000") {
return fmt.Errorf("native TLS refusal: %s", line)
}
}
}
if attempts != 2 {
return fmt.Errorf("native TLS attempt count is %d, want two", attempts)
}
for i := 0; i < 4; i++ {
select {
case err := <-results:
if err != nil {
return err
}
case <-ctx.Done():
return fmt.Errorf("native request count below four")
}
}
select {
case <-results:
return fmt.Errorf("unexpected extra request")
default:
}
fmt.Printf("GATEWAY_DIAGNOSTIC_WIRE=PASS transport=%s requests=4\n", os.Args[2])
return nil
}

func main() {
if err := run(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
Loading
Loading