#!/usr/bin/env bash
# T14 run 2 (re-run of run 1, which was confounded).
# Re-runnable: creates istio-vt-t14-r2, runs the observation, cleans up on exit.
#
# Corrections applied vs the literal spec (see harness-notes.md CORRECTION 2026-07-05):
#   1. istiod runs with --domain cluster.local (independent of kubeadm clusterDomain
#      homelab.local) -> Istio resource "host" fields for k8s services MUST use
#      <svc>.<ns>.svc.cluster.local, never *.svc.homelab.local, or outlierDetection
#      never attaches to the real Envoy cluster.
#   2. VirtualService fault.abort is a LOCAL Envoy HTTP-fault-filter reply -- it never
#      reaches outlier detection (proven in T05). This run drops the fault-abort
#      VirtualService entirely and instead triggers a REAL upstream 5xx by sending
#      request header 'x-set-response-status-code: 503' to a mendhak/http-https-echo
#      target (hashicorp/http-echo in the original spec cannot do this), so the
#      failure is a genuine upstream response that outlier detection actually counts.
#   3. curl targets use SHORT service names (matches Envoy vhost domains + resolves via
#      DNS search); only DestinationRule/VirtualService "host"/"hosts" fields use the
#      svc.cluster.local FQDN (the real registry name).
#   4. Trigger + observation loop + health_flags check are collapsed into ONE kubectl
#      exec/shell session (curl container shares pod netns with istio-proxy, so it can
#      hit localhost:15000 directly) to avoid a NEW confound: separate kubectl execs'
#      round-trip latency can burn enough of the 20s baseEjectionTime that the loop
#      starts after the ejection already naturally expired, producing a false
#      "all 200" (looks like panic-rescue, is actually just post-recovery traffic).

set -uo pipefail

NS="istio-vt-t14-r2"
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
RESULT="$HERE/result.txt"

cleanup() {
  echo "=== cleanup: deleting namespace $NS (--wait=false) ===" | tee -a "$RESULT"
  kubectl delete ns "$NS" --wait=false 2>&1 | tee -a "$RESULT"
}
trap cleanup EXIT

: > "$RESULT"

echo "=== create namespace ===" | tee -a "$RESULT"
kubectl create ns "$NS" 2>&1 | tee -a "$RESULT"
kubectl label ns "$NS" istio-injection=enabled --overwrite 2>&1 | tee -a "$RESULT"

echo "=== kubectl apply -f manifest.yaml ===" | tee -a "$RESULT"
kubectl apply -f "$HERE/manifest.yaml" 2>&1 | tee -a "$RESULT"

echo "=== wait pod/client Ready ===" | tee -a "$RESULT"
kubectl -n "$NS" wait --for=condition=Ready pod/client --timeout=90s 2>&1 | tee -a "$RESULT"
echo "=== wait deploy/single-target available ===" | tee -a "$RESULT"
kubectl -n "$NS" wait --for=condition=available deploy/single-target --timeout=60s 2>&1 | tee -a "$RESULT"
kubectl -n "$NS" get pods -o wide 2>&1 | tee -a "$RESULT"

echo "=== sanity: outlierDetection attached on DR host (svc.cluster.local) ===" | tee -a "$RESULT"
kubectl -n "$NS" exec client -c istio-proxy -- istioctl proxy-config cluster "client.$NS" \
  --fqdn "single-target.$NS.svc.cluster.local" -o json 2>&1 | grep -A6 outlierDetection | tee -a "$RESULT"

echo "=== baseline: normal request (no header) returns 200 before any fault ===" | tee -a "$RESULT"
kubectl -n "$NS" exec client -c curl -- curl -s -o /dev/null -w 'baseline=%{http_code}\n' \
  "http://single-target.$NS/" 2>&1 | tee -a "$RESULT"

echo "=== trigger + interleaved 18x (request+health_flags) in ONE exec session ===" | tee -a "$RESULT"
kubectl -n "$NS" exec client -c curl -- sh -c "
  echo \"trigger_at=\$(date -u +%Y-%m-%dT%H:%M:%S.%N)\"
  curl -s -o /dev/null -w 'trigger_status=%{http_code}\n' -H 'x-set-response-status-code: 503' http://single-target.$NS/
  i=1
  while [ \$i -le 18 ]; do
    hdrs=\$(curl -s -D - -o /dev/null --max-time 2 http://single-target.$NS/)
    has_upstream_hdr=\$(echo \"\$hdrs\" | grep -c 'x-envoy-upstream-service-time')
    flag=\$(curl -s localhost:15000/clusters | grep 'single-target.$NS.svc.cluster.local::10' | grep -o '80.*health_flags::[a-z_/]*')
    echo \"iter=\$i time=\$(date -u +%H:%M:%S.%N) status_line=[\$(echo \"\$hdrs\"|head -1)] has_upstream_hdr=\$has_upstream_hdr \$flag\"
    i=\$((i+1))
    sleep 1
  done
" 2>&1 | tee -a "$RESULT"

echo "=== post-window: confirm natural recovery (healthy + real upstream response) ===" | tee -a "$RESULT"
kubectl -n "$NS" exec client -c curl -- sh -c "
  curl -s localhost:15000/clusters | grep 'single-target.$NS.svc.cluster.local::10' | grep health_flags
  curl -s -D - http://single-target.$NS/
" 2>&1 | tee -a "$RESULT"

echo "=== done; cleanup runs via trap on exit ===" | tee -a "$RESULT"