#!/usr/bin/env bash
# T04: Phantom workload window (Arm1: abrupt kill / Arm2: preStop+readiness / Arm3: retries+outlierDetection)
#
# NOTE on manifest.yaml vs the original test spec: the DestinationRule.host and
# VirtualService.hosts for mitig-echo were changed from the FQDN
# "mitig-echo.istio-vt-t04.svc.homelab.local" to the short name "mitig-echo".
# This cluster's kubeadm dnsDomain is homelab.local, but Istio's own service
# registry/xDS naming always uses "cluster.local" regardless -- using the
# homelab.local FQDN as a VS/DR host creates an orphaned virtual host pointing
# at a nonexistent Envoy cluster (503 cluster_not_found). The short name
# resolves correctly against the real Istio-registered service in both cases.
# For the same reason, ALL curl targets below use short in-namespace service
# names (not the homelab.local FQDN) -- curling the homelab.local FQDN
# silently falls through to Envoy's PassthroughCluster (kube-proxy-level
# routing), bypassing Istio's EDS-based routing entirely.
set -euo pipefail

NS=istio-vt-t04
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

cleanup() {
  echo "[cleanup] deleting namespace $NS"
  kubectl delete namespace "$NS" --wait=false --ignore-not-found
}
trap cleanup EXIT

echo "[setup] creating namespace $NS with istio-injection=enabled"
kubectl create namespace "$NS"
kubectl label namespace "$NS" istio-injection=enabled --overwrite

echo "[setup] applying manifest"
kubectl apply -f "$SCRIPT_DIR/manifest.yaml"

echo "[wait] client pod ready"
kubectl -n "$NS" wait --for=condition=Ready pod/client --timeout=90s

echo "[wait] deployments available"
kubectl -n "$NS" wait --for=condition=available deploy/phantom-echo deploy/echo-graceful deploy/mitig-echo --timeout=120s

echo
echo "=== Arm 1: abrupt kill on phantom-echo (expect transient UF/UH failures) ==="
rm -f /tmp/t04_arm1.log
kubectl -n "$NS" exec client -c curl -- sh -c \
  'i=0; while [ $i -lt 400 ]; do curl -s -o /dev/null -w "%{http_code}\n" http://phantom-echo/; i=$((i+1)); done' \
  > /tmp/t04_arm1.log 2>&1 &
sleep 1
POD_A=$(kubectl -n "$NS" get pod -l app=phantom-echo -o jsonpath='{.items[0].metadata.name}')
kubectl -n "$NS" delete pod "$POD_A" --grace-period=0 --force
wait
echo "--- arm1 result summary ---"
sort /tmp/t04_arm1.log | uniq -c
echo "--- arm1 UF/UH count (istio-proxy access log) ---"
kubectl -n "$NS" logs client -c istio-proxy --since=30s | grep -E '"(UF|UH)"' | wc -l

echo
echo "=== Arm 2: graceful delete on echo-graceful (preStop+readiness; expect 0 failures) ==="
rm -f /tmp/t04_arm2.log
kubectl -n "$NS" exec client -c curl -- sh -c \
  'for i in $(seq 1 200); do curl -s -o /dev/null -w "%{http_code}\n" http://echo-graceful/; sleep 0.1; done' \
  > /tmp/t04_arm2.log 2>&1 &
sleep 1
POD_G=$(kubectl -n "$NS" get pod -l app=echo-graceful -o jsonpath='{.items[0].metadata.name}')
kubectl -n "$NS" delete pod "$POD_G"
wait
echo "--- arm2 result summary (expect only 200) ---"
sort /tmp/t04_arm2.log | uniq -c
echo "--- arm2 UC/UT/UH count (istio-proxy access log, expect 0) ---"
kubectl -n "$NS" logs client -c istio-proxy --since=30s | grep -E '"(UC|UT|UH)"' | wc -l

echo
echo "=== Arm 3: abrupt kill on mitig-echo (retries+outlierDetection; caller should see only 200) ==="
rm -f /tmp/t04_arm3.log
kubectl -n "$NS" exec client -c curl -- sh -c \
  'for i in $(seq 1 200); do curl -s -o /dev/null -w "%{http_code}\n" http://mitig-echo/; sleep 0.05; done' \
  > /tmp/t04_arm3.log 2>&1 &
sleep 1
POD_M=$(kubectl -n "$NS" get pod -l app=mitig-echo -o jsonpath='{.items[0].metadata.name}')
IP_M=$(kubectl -n "$NS" get pod "$POD_M" -o jsonpath='{.status.podIP}')
kubectl -n "$NS" delete pod "$POD_M" --grace-period=0 --force
wait
echo "--- arm3 caller-visible result summary (expect only 200) ---"
sort /tmp/t04_arm3.log | uniq -c
echo "--- arm3 internal UF/UC/UH count toward dead IP $IP_M (may be >0, absorbed by retries/outlier) ---"
kubectl -n "$NS" logs client -c istio-proxy --since=30s | grep "$IP_M" | grep -E '"(UF|UC|UH)"' | wc -l

echo
echo "[done] see trap for namespace cleanup"