#!/usr/bin/env bash
# T52: DestinationRule connectionPool.tcp.maxConnections circuit breaker (UO/upstream_cx_overflow) reproduction
#
# IMPORTANT ENVIRONMENT NOTE (discovered during verification, see result.txt for full detail):
# On this cluster, Istio's internal Service registry hostnames use the DEFAULT ".svc.cluster.local"
# suffix regardless of the actual kubeadm/CoreDNS clusterDomain ("homelab.local"). A DestinationRule
# whose spec.host uses ".svc.homelab.local" (the real cluster DNS domain) will NOT match any known
# mesh service (istioctl analyze: IST0174) and its trafficPolicy will silently not apply. Worse, a
# client request whose Host header/URL uses ".svc.homelab.local" also fails to match Envoy's HTTP
# virtual-host domain list and falls through to PassthroughCluster under ALLOW_ANY outbound policy,
# bypassing the circuit breaker entirely even though DNS resolves correctly to the right pod.
#
# This script therefore uses the CORRECT/matching hostname forms:
#   - DestinationRule.host  = echo.<ns>.svc.cluster.local   (matches Istio's actual registry hostname)
#   - client curl target    = http://echo/                  (short name; avoids the domain-suffix issue,
#                                                              still resolves fine via k8s DNS search path)
#
# Also note: this cluster's mesh-wide proxyStatsMatcher excludes ALL outbound-cluster stats (including
# upstream_cx_overflow) from /stats and /stats/prometheus - only cluster.xds-grpc control-plane stats are
# exposed. So circuit-breaker overflow is verified here via HTTP response codes + access-log response flags
# (TEXT format: bare "UO" token + "upstream_reset_before_response_started{overflow}"), NOT via /stats counters.

set -euo pipefail

NS=istio-vt-t52
WORKDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

cleanup() {
  echo "--- cleanup: deleting namespace $NS ---"
  kubectl delete namespace "$NS" --wait=false --ignore-not-found
}
trap cleanup EXIT

echo "--- create namespace ---"
kubectl create namespace "$NS"
kubectl label namespace "$NS" istio-injection=enabled

echo "--- apply manifest (client, echo deploy/svc, DestinationRule) ---"
kubectl apply -f "$WORKDIR/manifest.yaml"

echo "--- wait for workloads ready ---"
kubectl -n "$NS" wait --for=condition=Ready pod/client --timeout=90s
kubectl -n "$NS" rollout status deploy/echo --timeout=90s

echo "--- FIX: patch DestinationRule host to match Istio's actual registry hostname (see note above) ---"
kubectl -n "$NS" patch destinationrule echo-connpool-limit --type merge \
  -p '{"spec":{"host":"echo.'"$NS"'.svc.cluster.local"}}'
sleep 2

echo "--- verify circuit breaker config attached (expect maxConnections=2) ---"
istioctl -n "$NS" proxy-config cluster "client.$NS" --fqdn "echo.$NS.svc.cluster.local" -o json \
  | python3 -c "import json,sys; d=json.load(sys.stdin); [print(c['name'], c.get('circuitBreakers')) for c in d]"

echo "--- before: outbound_cx_overflow stat (expected to be absent - see mesh stats-matcher note) ---"
kubectl -n "$NS" exec client -c istio-proxy -- curl -s localhost:15000/stats \
  | grep 'outbound|80||echo.*upstream_cx_overflow' || echo "(no matching stat line - expected on this cluster)"

echo "--- fire 40 concurrent requests (short name to stay in-mesh, avoid PassthroughCluster) ---"
kubectl -n "$NS" exec client -c curl -- sh -c \
  'for i in $(seq 1 40); do curl -s -o /dev/null -w "%{http_code} " --max-time 3 http://echo/ & done; wait; echo'

echo "--- after: outbound_cx_overflow stat (still expected absent on this cluster) ---"
kubectl -n "$NS" exec client -c istio-proxy -- curl -s localhost:15000/stats \
  | grep 'outbound|80||echo.*upstream_cx_overflow' || echo "(no matching stat line - expected on this cluster)"

echo "--- UO response-flag count in access log (TEXT format: match bare ' UO ' token, not JSON '\"UO\"') ---"
kubectl -n "$NS" logs client -c istio-proxy --since=1m | grep -c ' UO ' || true

echo "--- sample UO lines ---"
kubectl -n "$NS" logs client -c istio-proxy --since=1m | grep ' UO ' | head -5 || true

echo "--- done ---"