ci(kubernetes): stabilize envoy e2e setup

Signed-off-by: Taylor Mutch <taylormutch@gmail.com>

Author: TaylorMutch

.agents/skills/helm-dev-environment/SKILL.md

@@ -197,9 +197,9 @@ Service rather than `svc/openshell`. That keeps client traffic on the same path
 as a real reverse proxy while gateway pods rotate behind it:
 
 ```bash
-KUBECONFIG=kubeconfig kubectl -n openshell get svc \
+KUBECONFIG=kubeconfig kubectl get svc -A \
   -l gateway.envoyproxy.io/owning-gateway-name=openshell
-KUBECONFIG=kubeconfig kubectl -n openshell port-forward \
+KUBECONFIG=kubeconfig kubectl -n <envoy-service-namespace> port-forward \
   svc/<envoy-service-name> 8080:80
 openshell gateway add http://127.0.0.1:8080 --name openshell --local
 ```

e2e/with-kube-gateway.sh

@@ -143,40 +143,57 @@ install_envoy_gateway() {
 }
 
 wait_for_envoy_service() {
-  local svc=""
+  local svc_ref=""
+  local svc_namespace=""
 
   for _ in $(seq 1 60); do
-    svc="$(kctl -n "${NAMESPACE}" get svc \
-      -l "gateway.envoyproxy.io/owning-gateway-name=${RELEASE_NAME}" \
-      -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
-    if [ -n "${svc}" ]; then
-      printf '%s\n' "${svc}"
-      return 0
+    svc_ref="$(kctl get svc -A \
+      -l "gateway.envoyproxy.io/owning-gateway-name=${RELEASE_NAME},gateway.envoyproxy.io/owning-gateway-namespace=${NAMESPACE}" \
+      -o jsonpath='{range .items[0]}{.metadata.namespace}{"/"}{.metadata.name}{end}' \
+      2>/dev/null || true)"
+    if [ -n "${svc_ref}" ]; then
+      svc_namespace="${svc_ref%%/*}"
+      if kctl -n "${svc_namespace}" wait --for=condition=Ready pod \
+        -l "gateway.envoyproxy.io/owning-gateway-name=${RELEASE_NAME},gateway.envoyproxy.io/owning-gateway-namespace=${NAMESPACE}" \
+        --timeout=5s >/dev/null 2>&1; then
+        printf '%s\n' "${svc_ref}"
+        return 0
+      fi
     fi
     sleep 2
   done
 
-  echo "ERROR: Envoy proxy Service for Gateway ${RELEASE_NAME} was not created." >&2
-  kctl -n "${NAMESPACE}" get gateway,grpcroute,svc -o wide >&2 || true
+  echo "ERROR: Envoy proxy Service for Gateway ${RELEASE_NAME} was not ready." >&2
+  kctl -n "${NAMESPACE}" get gateway,grpcroute -o wide >&2 || true
+  kctl get svc -A \
+    -l "gateway.envoyproxy.io/owning-gateway-name=${RELEASE_NAME},gateway.envoyproxy.io/owning-gateway-namespace=${NAMESPACE}" \
+    -o wide >&2 || true
+  kctl get pods -A \
+    -l "gateway.envoyproxy.io/owning-gateway-name=${RELEASE_NAME},gateway.envoyproxy.io/owning-gateway-namespace=${NAMESPACE}" \
+    -o wide >&2 || true
   return 1
 }
 
 start_gateway_portforward() {
   local elapsed=0
   local pf_timeout=30
   local target_port=8080
+  local target_namespace="${NAMESPACE}"
   local target_service="${RELEASE_NAME}"
+  local target_service_ref=""
 
   LOCAL_PORT="$(e2e_pick_port)"
   if use_envoy_gateway; then
-    target_service="$(wait_for_envoy_service)"
+    target_service_ref="$(wait_for_envoy_service)"
+    target_namespace="${target_service_ref%%/*}"
+    target_service="${target_service_ref#*/}"
     target_port=80
-    echo "Starting kubectl port-forward svc/${target_service} ${LOCAL_PORT}:${target_port} (Envoy Gateway)..."
+    echo "Starting kubectl port-forward -n ${target_namespace} svc/${target_service} ${LOCAL_PORT}:${target_port} (Envoy Gateway)..."
   else
     echo "Starting kubectl port-forward svc/${target_service} ${LOCAL_PORT}:${target_port}..."
   fi
 
-  kctl -n "${NAMESPACE}" port-forward "svc/${target_service}" \
+  kctl -n "${target_namespace}" port-forward "svc/${target_service}" \
     "${LOCAL_PORT}:${target_port}" >"${PORTFORWARD_LOG}" 2>&1 &
   PORTFORWARD_PID=$!
 
@@ -198,6 +215,21 @@ start_gateway_portforward() {
   return 1
 }
 
+stop_gateway_portforward() {
+  [ -n "${PORTFORWARD_PID}" ] || return 0
+
+  kill "${PORTFORWARD_PID}" >/dev/null 2>&1 || true
+  for _ in $(seq 1 10); do
+    if ! kill -0 "${PORTFORWARD_PID}" >/dev/null 2>&1; then
+      break
+    fi
+    sleep 0.5
+  done
+  kill -KILL "${PORTFORWARD_PID}" >/dev/null 2>&1 || true
+  wait "${PORTFORWARD_PID}" >/dev/null 2>&1 || true
+  PORTFORWARD_PID=""
+}
+
 cleanup_postgres_fixture() {
   local secret_name="$1"
 
@@ -216,10 +248,7 @@ cleanup_postgres_fixture() {
 cleanup() {
   local exit_code=$?
 
-  if [ -n "${PORTFORWARD_PID}" ]; then
-    kill "${PORTFORWARD_PID}" >/dev/null 2>&1 || true
-    wait "${PORTFORWARD_PID}" >/dev/null 2>&1 || true
-  fi
+  stop_gateway_portforward
 
   if [ "${exit_code}" -ne 0 ] && [ -n "${KUBE_CONTEXT}" ] && [ -n "${NAMESPACE}" ]; then
     if command -v kubectl >/dev/null 2>&1 \
@@ -242,20 +271,6 @@ cleanup() {
     fi
   fi
 
-  if [ "${EXTERNAL_PG_FIXTURE_DEPLOYED}" = "1" ]; then
-    cleanup_postgres_fixture "${EXTERNAL_PG_FIXTURE_SECRET}"
-  fi
-
-  if [ "${ENVOY_GATEWAY_CONFIG_APPLIED}" = "1" ] && [ -n "${KUBE_CONTEXT}" ]; then
-    if command -v kubectl >/dev/null 2>&1; then
-      kctl -n "${NAMESPACE}" delete backendtrafficpolicy.gateway.envoyproxy.io \
-        openshell-grpc-timeouts --ignore-not-found >/dev/null 2>&1 || true
-      kctl delete gatewayclass.gateway.networking.k8s.io eg \
-        --ignore-not-found >/dev/null 2>&1 || true
-    fi
-    ENVOY_GATEWAY_CONFIG_APPLIED=0
-  fi
-
   if [ "${HELM_INSTALLED}" = "1" ] && [ -n "${KUBE_CONTEXT}" ] && [ -n "${NAMESPACE}" ]; then
     if command -v helm >/dev/null 2>&1; then
       helmctl uninstall "${RELEASE_NAME}" --namespace "${NAMESPACE}" --wait \
@@ -269,6 +284,21 @@ cleanup() {
     fi
   fi
 
+  if [ "${EXTERNAL_PG_FIXTURE_DEPLOYED}" = "1" ]; then
+    cleanup_postgres_fixture "${EXTERNAL_PG_FIXTURE_SECRET}"
+  fi
+
+  if [ "${ENVOY_GATEWAY_CONFIG_APPLIED}" = "1" ] && [ -n "${KUBE_CONTEXT}" ]; then
+    if command -v kubectl >/dev/null 2>&1; then
+      kctl -n "${NAMESPACE}" delete backendtrafficpolicy.gateway.envoyproxy.io \
+        openshell-grpc-timeouts --ignore-not-found --wait=false \
+        >/dev/null 2>&1 || true
+      kctl delete gatewayclass.gateway.networking.k8s.io eg \
+        --ignore-not-found --wait=false >/dev/null 2>&1 || true
+    fi
+    ENVOY_GATEWAY_CONFIG_APPLIED=0
+  fi
+
   if [ "${ENVOY_HELM_INSTALLED}" = "1" ] && [ -n "${KUBE_CONTEXT}" ]; then
     if command -v helm >/dev/null 2>&1; then
       helmctl uninstall "${ENVOY_RELEASE_NAME}" --namespace "${ENVOY_NAMESPACE}" \
@@ -296,11 +326,7 @@ trap cleanup EXIT
 # --- DB-scenario helpers (used only when OPENSHELL_E2E_KUBE_DB_SCENARIOS=1) ---
 
 scenario_stop_portforward() {
-  if [ -n "${PORTFORWARD_PID}" ]; then
-    kill "${PORTFORWARD_PID}" >/dev/null 2>&1 || true
-    wait "${PORTFORWARD_PID}" >/dev/null 2>&1 || true
-    PORTFORWARD_PID=""
-  fi
+  stop_gateway_portforward
 }
 
 scenario_cleanup_release() {