#!/usr/bin/env bash
# run-scenario.sh — DR connectionPool 필드별 "발동" 재현 하네스.
#
# 전제: setup.sh 로 랩이 떠 있어야 함. DR 매니페스트는 dr-scenarios/ 에 있다.
#
# 핵심 계측 원리(dns-lab 교훈): Envoy cx/rq 카운터는 프록시 기동 이래 "누적"이라 절대값은
# 이전 run 이 이월된다. 따라서 판정은 항상 "이 run 내 before→after 델타"로 격리한다.
# 시나리오마다: DR apply → 3s 반영 대기 → before 스냅샷 → 부하 → after 스냅샷 → 델타 출력 → 리포트.
#
# usage: bash run-scenario.sh <s1|s2|s3|s4|s5|s6|s7>
set -uo pipefail

SC="${1:?usage: run-scenario.sh <s1|s2|s3|s4|s5|s6|s7>}"
CTX="${CTX:-$(kubectl config current-context)}"; NS="${NS:-conn-lab}"
DIR="$(cd "$(dirname "$0")" && pwd)"; DRD="$DIR/dr-scenarios"
K="kubectl --context=$CTX -n $NS"
STAMP="$(date +%Y-%m-%d_%H%M%S)"
OUT="$DIR/reports/${STAMP}_${SC}.md"; mkdir -p "$(dirname "$OUT")" "$DIR/tmp"
T="$DIR/tmp"

# ── 스냅샷/추출 헬퍼 ──────────────────────────────────────────────────────────
snap(){ # $1 filter  $2 outfile — client istio-proxy stats 를 필터해 저장
  $K exec deploy/client -c istio-proxy -- pilot-agent request GET "stats?filter=$1" 2>/dev/null > "$2"
}
getv(){ # $1 file  $2 metric-suffix — '.<metric>: <value>' 의 value(없으면 0)
  awk -v m="$2" 'index($0, "." m ": "){v=$NF} END{print (v==""?0:v)}' "$1"
}
delta(){ # $1 metric  $2 before-file  $3 after-file
  local b a; b=$(getv "$2" "$1"); a=$(getv "$3" "$1")
  printf "    %-42s %8s -> %-8s = delta %s\n" "$1" "$b" "$a" "$((a-b))"
}
point(){ # $1 metric  $2 file — 게이지 등 point 값
  printf "    %-42s = %s\n" "$1" "$(getv "$2" "$1")"
}
apply_dr(){ # 시나리오 DR 만 남기고 나머지 제거(스택 방지 — 같은 host 다중 DR 회피)
  $K delete destinationrule conn-dr conn-blackhole-dr --ignore-not-found >/dev/null 2>&1
  $K apply -f "$1" >/dev/null
}

{
echo "# DR connectionPool 발동 재현 — $SC"
echo; echo "**Date:** $STAMP · **ns:** $NS · **cluster:** $CTX"
echo "> 판정은 이 run 내 before→after 델타로 격리(cx/rq 카운터는 누적)."
echo

case "$SC" in
# ─────────────────────────────────────────────────────────────────────────────
s1) # maxConnections:1 → upstream_cx_overflow, gauge cx_open
  echo "## S1 maxConnections:1 — 부하 fortio -c 4 -qps 0 -n 40 (?delay=200ms)"
  apply_dr "$DRD/s1-maxconnections.yaml"; sleep 3
  snap "backend.conn-lab" "$T/b"
  $K exec deploy/client -c fortio -- fortio load -quiet -c 4 -qps 0 -n 40 "http://backend:8080/?delay=200ms" > "$T/load" 2>&1 &
  LPID=$!
  sleep 2; echo "### mid-load 게이지(부하 중)"; echo '```'; snap "backend.conn-lab" "$T/mid"; point circuit_breakers.default.cx_open "$T/mid"; echo '```'
  wait $LPID
  snap "backend.conn-lab" "$T/a"
  echo; echo "### 델타"; echo '```'
  delta upstream_cx_overflow "$T/b" "$T/a"
  delta upstream_cx_total    "$T/b" "$T/a"
  delta upstream_rq_total    "$T/b" "$T/a"
  echo "  -- fortio --"; grep -E 'Sockets used|Code |All done' "$T/load"
  echo '```'
  echo "판정: upstream_cx_overflow delta > 0 이면 커넥션 서킷브레이커 발동."
  ;;
# ─────────────────────────────────────────────────────────────────────────────
s2) # maxConnections:1 + http1MaxPendingRequests:1 → upstream_rq_pending_overflow, 503/UO
  echo "## S2 maxConnections:1 + http1MaxPendingRequests:1 — 부하 fortio -c 3 -qps 0 -n 30 (?delay=200ms)"
  apply_dr "$DRD/s2-maxpending.yaml"; sleep 3
  snap "backend.conn-lab" "$T/b"
  $K exec deploy/client -c fortio -- fortio load -quiet -c 3 -qps 0 -n 30 "http://backend:8080/?delay=200ms" > "$T/load" 2>&1 &
  LPID=$!
  sleep 1.5; echo "### mid-load 게이지"; echo '```'; snap "backend.conn-lab" "$T/mid"; point circuit_breakers.default.rq_pending_open "$T/mid"; echo '```'
  wait $LPID
  snap "backend.conn-lab" "$T/a"
  echo; echo "### 델타"; echo '```'
  delta upstream_rq_pending_overflow "$T/b" "$T/a"
  delta upstream_cx_overflow         "$T/b" "$T/a"
  delta upstream_rq_total            "$T/b" "$T/a"
  echo "  -- fortio (503 = UO circuit-break) --"; grep -E 'Sockets used|Code |All done' "$T/load"
  echo '```'
  echo "판정: upstream_rq_pending_overflow delta > 0 + fortio Code 503 존재."
  ;;
# ─────────────────────────────────────────────────────────────────────────────
s3) # maxRequestsPerConnection:1 → upstream_cx_max_requests, cx_total turnover
  echo "## S3 maxRequestsPerConnection:1 — 부하 fortio -c 1 -qps 0 -n 20 -keepalive"
  apply_dr "$DRD/s3-maxrequestsperconn.yaml"; sleep 3
  snap "backend.conn-lab" "$T/b"
  $K exec deploy/client -c fortio -- fortio load -quiet -c 1 -qps 0 -n 20 -keepalive "http://backend:8080/" > "$T/load" 2>&1
  snap "backend.conn-lab" "$T/a"
  echo; echo "### 델타"; echo '```'
  delta upstream_cx_max_requests "$T/b" "$T/a"
  delta upstream_cx_total        "$T/b" "$T/a"
  delta upstream_rq_total        "$T/b" "$T/a"
  echo "  -- fortio (Sockets used ≈ 커넥션 turnover) --"; grep -E 'Sockets used|Code |All done' "$T/load"
  echo '```'
  echo "판정: upstream_cx_max_requests delta ≈ 요청 수, cx_total 동반 증가(거부 아닌 turnover)."
  ;;
# ─────────────────────────────────────────────────────────────────────────────
s4) # http.idleTimeout:2s → upstream_cx_idle_timeout +1
  echo "## S4 http.idleTimeout:2s — 요청 1회(keepalive) → 5s 대기 → idle 종료"
  apply_dr "$DRD/s4-http-idletimeout.yaml"; sleep 3
  snap "backend.conn-lab" "$T/b"
  $K exec deploy/client -c fortio -- fortio load -quiet -c 1 -qps 0 -n 1 -keepalive "http://backend:8080/" > "$T/load" 2>&1
  echo "### 요청 후 5s idle 대기…"; sleep 5
  snap "backend.conn-lab" "$T/a"
  echo; echo "### 델타"; echo '```'
  delta upstream_cx_idle_timeout "$T/b" "$T/a"
  delta upstream_cx_total        "$T/b" "$T/a"
  echo '```'
  echo "판정: upstream_cx_idle_timeout delta >= 1(idle 2s 초과로 풀 커넥션 종료)."
  ;;
# ─────────────────────────────────────────────────────────────────────────────
s5) # tcp.maxConnectionDuration:5s → upstream_cx_max_duration_reached >= 1
  echo "## S5 tcp.maxConnectionDuration:5s — keepalive 10s 부하 → 5s 시점 수명 도달"
  apply_dr "$DRD/s5-maxconnduration.yaml"; sleep 3
  snap "backend.conn-lab" "$T/b"
  $K exec deploy/client -c fortio -- fortio load -quiet -c 1 -qps 10 -t 10s -keepalive "http://backend:8080/" > "$T/load" 2>&1
  snap "backend.conn-lab" "$T/a"
  echo; echo "### 델타"; echo '```'
  delta upstream_cx_max_duration_reached "$T/b" "$T/a"
  delta upstream_cx_total                "$T/b" "$T/a"
  echo "  -- fortio --"; grep -E 'Sockets used|Code |All done' "$T/load"
  echo '```'
  echo "판정: upstream_cx_max_duration_reached delta >= 1(5s 수명 도달 후 graceful close·재수립)."
  ;;
# ─────────────────────────────────────────────────────────────────────────────
s6) # connectTimeout:1s (blackhole) → upstream_cx_connect_timeout, 503, UF
  echo "## S6 connectTimeout:1s (blackhole.lab.internal) — fortio -c 1 -n 5 -timeout 5s"
  apply_dr "$DRD/s6-connecttimeout.yaml"; sleep 3
  snap blackhole "$T/b"
  $K exec deploy/client -c fortio -- fortio load -quiet -c 1 -qps 0 -n 5 -timeout 5s "http://blackhole.lab.internal/" > "$T/load" 2>&1
  snap blackhole "$T/a"
  echo; echo "### 델타"; echo '```'
  delta upstream_cx_connect_timeout "$T/b" "$T/a"
  delta upstream_cx_connect_fail    "$T/b" "$T/a"
  delta upstream_cx_total           "$T/b" "$T/a"
  echo "  -- fortio --"; grep -E 'Sockets used|Code |All done|error' "$T/load"
  echo "  -- client istio-proxy 액세스 로그(response flag; UF 기대) --"
  $K logs deploy/client -c istio-proxy --since=40s 2>/dev/null | grep blackhole | tail -5
  echo '```'
  echo "판정: upstream_cx_connect_timeout delta > 0 + 503 + 액세스로그 UF."
  ;;
# ─────────────────────────────────────────────────────────────────────────────
s7) # config 반영 검증(발동 아님) — istioctl pc cluster + config_dump
  CL="deploy/client.$NS"
  DOMAIN="$($K get svc backend -o jsonpath='{.metadata.name}' >/dev/null 2>&1 && \
            kubectl --context=$CTX -n kube-system get cm coredns -o jsonpath='{.data.Corefile}' 2>/dev/null \
            | grep -oE 'kubernetes[[:space:]]+[^ ]+' | awk '{print $2}')"
  DOMAIN="${DOMAIN:-cluster.local}"
  FQDN="backend.$NS.svc.$DOMAIN"
  dump(){ istioctl --context="$CTX" pc cluster "$CL" --fqdn "$FQDN" --port 8080 -o json 2>/dev/null; }

  echo "## S7 config 반영 검증 — FQDN=$FQDN (cluster domain 자동발견=$DOMAIN)"
  echo

  echo "### S7-full: tcpKeepalive + maxRetries:2 + h2UpgradePolicy:UPGRADE + maxConcurrentStreams:64"
  apply_dr "$DRD/s7-full.yaml"; sleep 4
  dump > "$T/s7full.json"
  echo '```'
  echo "① upstream_connection_options.tcp_keepalive:"
  jq -c '.[0].upstreamConnectionOptions.tcpKeepalive // "ABSENT"' "$T/s7full.json"
  echo "② circuit_breakers.thresholds (maxRetries=2 반영 + 생략 필드 주입값 §5-2):"
  jq -c '.[0].circuitBreakers.thresholds' "$T/s7full.json"
  echo "③④ explicit_http_config.http2_protocol_options (maxConcurrentStreams=64 반영):"
  jq -c '.[0].typedExtensionProtocolOptions["envoy.extensions.upstreams.http.v3.HttpProtocolOptions"].explicitHttpConfig.http2ProtocolOptions // "ABSENT"' "$T/s7full.json"
  echo '```'
  echo

  echo "### S7c: h2UpgradePolicy:UPGRADE 만 (maxConcurrentStreams 미설정 §5-1 실측)"
  apply_dr "$DRD/s7c-h2-nomcs.yaml"; sleep 4
  dump > "$T/s7c.json"
  echo '```'
  echo "http2_protocol_options (maxConcurrentStreams 생략 시 값):"
  jq -c '.[0].typedExtensionProtocolOptions["envoy.extensions.upstreams.http.v3.HttpProtocolOptions"].explicitHttpConfig.http2ProtocolOptions // "ABSENT"' "$T/s7c.json"
  echo "circuit_breakers.thresholds (connectionPool.http 만, CB 4종 전부 생략 시 주입값):"
  jq -c '.[0].circuitBreakers.thresholds' "$T/s7c.json"
  echo '```'
  echo

  echo "### S7b: useClientProtocol:true (use_downstream_protocol_config 확인, h2UpgradePolicy 무력화)"
  apply_dr "$DRD/s7b-useclientprotocol.yaml"; sleep 4
  dump > "$T/s7b.json"
  echo '```'
  echo "use_downstream_protocol_config:"
  jq -c '.[0].typedExtensionProtocolOptions["envoy.extensions.upstreams.http.v3.HttpProtocolOptions"].useDownstreamProtocolConfig // "ABSENT"' "$T/s7b.json"
  echo "explicit http2 승격 잔존 여부(useClientProtocol 이 무력화하면 ABSENT 기대):"
  jq -c '.[0].typedExtensionProtocolOptions["envoy.extensions.upstreams.http.v3.HttpProtocolOptions"].explicitHttpConfig // "ABSENT"' "$T/s7b.json"
  echo '```'
  echo "판정: config_dump 로 4개 필드 반영 위치 확인 + 생략 필드 주입값(§5) 기록."
  ;;
*) echo "unknown scenario: $SC (s1..s7)"; exit 1 ;;
esac
} | tee "$OUT"

echo; echo ">> 리포트 저장: $OUT"