#!/usr/bin/env bash
# T56 (re-run, attempt 2): ephemeral 포트 고갈(짧은 연결 반복 + TIME_WAIT 점유)로
# connect가 실패하면 UF/upstream_cx_connect_fail이 남는지 확인.
#
# CORRECTIONS applied on top of the original spec (see harness-notes.md
# CORRECTION 2026-07-05 + this run's own findings):
#   1) net.ipv4.ip_local_port_range is kubelet-safe-listed -> set via pod
#      spec.securityContext.sysctls at pod CREATION time (applies pod-wide,
#      shared netns with istio-proxy) instead of a runtime `sysctl -w`, which
#      run 1 showed fails with "Read-only file system" regardless of NET_ADMIN.
#   2) Istio's internal cluster/route registry always uses <svc>.<ns>.svc.cluster.local
#      (istiod --domain=cluster.local) regardless of the cluster's real kubelet
#      clusterDomain (homelab.local). curl MUST use the SHORT service name
#      (http://echo.<ns>/) so the Host header matches an Envoy vhost domain;
#      the literal *.svc.homelab.local FQDN falls through to PassthroughCluster.
#   3) /stats (plaintext) does not expose per-cluster upstream_cx_connect_fail
#      on this cluster by default (Istio's built-in stats-inclusion matcher only
#      keeps the istiod xds-grpc bootstrap cluster there) -- verified empirically,
#      no custom proxyStatsMatcher override in the istio-system mesh ConfigMap.
#      The SAME underlying Envoy counter IS exposed, unfiltered, via the
#      /clusters admin endpoint (`<cluster>::<host>::cx_connect_fail::<n>`),
#      which this script uses as the authoritative before/after signal.
#   4) A strictly SEQUENTIAL loop (as in the original spec's CMD4) produces ZERO
#      connect failures on this node because net.ipv4.tcp_tw_reuse=2 (kernel
#      default) lets a single in-flight client reuse a TIME_WAIT 4-tuple for the
#      next connect() to the same destination. To actually stress the narrowed
#      20-port pod-wide budget, a CONCURRENT burst is required; this script runs
#      the spec's 200x sequential loop first (faithful reproduction, expect 0
#      failures) and then a 60x concurrent burst as a diagnostic to determine
#      whether the exhaustion mechanism can manifest at all on this cluster.
set -uo pipefail  # no -e: we expect some commands (curl bursts) to "fail" by design

NS=istio-vt-t56-r2
WORKDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

cleanup() {
  echo "[cleanup] deleting namespace ${NS}"
  kubectl delete namespace "${NS}" --wait=false --ignore-not-found
}
trap cleanup EXIT

echo "[setup] creating namespace ${NS}"
kubectl create namespace "${NS}"
kubectl label namespace "${NS}" istio-injection=enabled

echo "[apply] manifest.yaml"
kubectl apply -f "${WORKDIR}/manifest.yaml"

echo "[wait] port-exhaust-client / client Ready, echo Available"
kubectl -n "${NS}" wait --for=condition=Ready pod/port-exhaust-client --timeout=90s
kubectl -n "${NS}" wait --for=condition=Ready pod/client --timeout=90s
kubectl -n "${NS}" wait --for=condition=Available deployment/echo --timeout=90s

echo "[verify] pod-wide ip_local_port_range (via securityContext.sysctls, no runtime write needed)"
kubectl -n "${NS}" exec port-exhaust-client -c curl -- cat /proc/sys/net/ipv4/ip_local_port_range
kubectl -n "${NS}" exec port-exhaust-client -c istio-proxy -- cat /proc/sys/net/ipv4/ip_local_port_range

echo "[before] /clusters cx_connect_fail / cx_total for outbound|80||echo.${NS}.svc.cluster.local"
kubectl -n "${NS}" exec port-exhaust-client -c istio-proxy -- curl -s localhost:15000/clusters \
  | grep "outbound|80||echo.${NS}.svc.cluster.local" | grep -E "cx_connect_fail|cx_total"

echo "[cmd4] 200x sequential short-lived curl against SHORT name http://echo.${NS}/"
kubectl -n "${NS}" exec port-exhaust-client -c curl -- sh -c '
  ok=0; fail=0
  for i in $(seq 1 200); do
    if curl -s -o /dev/null --max-time 1 -H "Connection: close" http://echo.'"${NS}"'/; then
      ok=$((ok+1))
    else
      fail=$((fail+1))
    fi
  done
  echo "sequential: ok=$ok fail=$fail"
'

echo "[diag] 60x CONCURRENT short-lived curl (forces real contention on the 20-port pod-wide budget)"
kubectl -n "${NS}" exec port-exhaust-client -c curl -- sh -c '
  for i in $(seq 1 60); do
    ( curl -s -o /dev/null -w "%{http_code} %{exitcode}\n" --max-time 2 -H "Connection: close" http://echo.'"${NS}"'/ || echo CURLFAIL ) &
  done
  wait
'

echo "[cmd5] TIME_WAIT count after both loops"
kubectl -n "${NS}" exec port-exhaust-client -c istio-proxy -- ss -tan state time-wait | wc -l

echo "[after] /clusters cx_connect_fail / cx_total"
kubectl -n "${NS}" exec port-exhaust-client -c istio-proxy -- curl -s localhost:15000/clusters \
  | grep "outbound|80||echo.${NS}.svc.cluster.local" | grep -E "cx_connect_fail|cx_total"

echo "[cmd7] UF count in istio-proxy access log"
kubectl -n "${NS}" logs port-exhaust-client -c istio-proxy --since=5m | grep -c '"UF"' || true

echo "[done] see result.txt for the full annotated transcript of the actual run"