#!/usr/bin/env bash
# T47 RUN 2: PASSTHROUGH egress gateway 2-hop SNI routing / L7 visibility loss / retries-silently-ignored
#
# Re-runnable verification script. Applies the corrected FQDN adaptation rule on top of the
# original spec (see harness-notes.md CORRECTION 2026-07-05): the DestinationRule host and the
# "mesh"-gateway VirtualService's tls-route destination host for the real k8s hop-gw Service use
# hop-gw.<ns>.svc.cluster.local (Istio's internal service-registry FQDN), NOT
# hop-gw.<ns>.svc.homelab.local (the cluster's real kubeadm clusterDomain). ServiceEntry hosts,
# Gateway hosts, VS sniHosts, and the at-gw VS's route-to-mock destination remain literal
# mock.istio-verify-ext.svc.homelab.local, per the "ServiceEntry hosts are literal" rule.
#
# RESULT (see result.txt for full investigation): this fix corrects the CDS/EDS-level defect from
# attempt1 (hop-gw's cluster now exists with a live healthy endpoint) but pass_criteria STILL
# fails: hop-gw structurally never receives any traffic (istio_tcp_connections_opened_total stays
# 0 across repeated 200 requests, and hop-gw scaled to 0 replicas has zero effect on the request
# outcome). Root cause (new in this run): mock.istio-verify-ext.svc.homelab.local DNS-resolves to
# the real ClusterIP of the k8s Service `mock`, which already has its own dedicated per-ClusterIP
# listener in Envoy (10.250.183.220_443) with a single default filter chain routing straight to
# mock's real cluster -- this takes precedence over the wildcard 0.0.0.0_443 listener where the
# "mesh"-gateway SNI-matched route to hop-gw lives, so that route is never evaluated. Using an
# in-cluster k8s Service as the ServiceEntry's "external" stand-in defeats this mechanism
# regardless of the DR/VS domain fix.
set -euo pipefail

NS="istio-vt-t47-r2"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

cleanup() {
  echo "--- cleanup: deleting namespace ${NS} ---"
  kubectl delete namespace "${NS}" --wait=false --ignore-not-found
}
trap cleanup EXIT

echo "--- create namespace ${NS} (istio-injection=enabled) ---"
kubectl create namespace "${NS}"
kubectl label namespace "${NS}" istio-injection=enabled

echo "--- apply manifest (corrected DR/VS host = hop-gw.<ns>.svc.cluster.local) ---"
kubectl apply -f "${SCRIPT_DIR}/manifest.yaml"

echo "--- wait for workloads Ready ---"
kubectl -n "${NS}" wait --for=condition=Ready pod/client --timeout=90s
kubectl -n "${NS}" rollout status deploy/echo --timeout=90s
kubectl -n "${NS}" rollout status deploy/hop-gw --timeout=90s
kubectl -n "${NS}" get pods -o wide

echo "--- CMD1: client -> mock https, expect twohop_https=200 ---"
kubectl -n "${NS}" exec client -c curl -- curl -sk -o /dev/null -w 'twohop_https=%{http_code}\n' \
  https://mock.istio-verify-ext.svc.homelab.local/

echo "--- CMD2: hop-gw access log tail (expect L4-only format IF traffic actually transits hop-gw) ---"
kubectl -n "${NS}" logs deploy/hop-gw -c istio-proxy --since=1m | tail -10

echo "--- CMD3: client listener filter names on port 443 ---"
istioctl proxy-config listener "client.${NS}" --port 443 -o json | jq '.[0].filterChains[0].filters[].name'

echo "--- CMD4: hop-gw istio_tcp_connections_opened_total count (expect >0 if hop-gw truly in path) ---"
kubectl -n "${NS}" exec deploy/hop-gw -c istio-proxy -- curl -s localhost:15000/stats/prometheus \
  | grep -c '^istio_tcp_connections_opened_total' || true

echo "--- CMD5: hop-gw istio_requests_total count (expect 0: no L7 visibility for TLS passthrough) ---"
kubectl -n "${NS}" exec deploy/hop-gw -c istio-proxy -- curl -s localhost:15000/stats/prometheus \
  | grep -c '^istio_requests_total' || true

echo "--- CMD6: client HTTP route count for mock host (expect 0: no HTTP route for TLS-passthrough host) ---"
istioctl proxy-config route "client.${NS}" -o json \
  | jq '[.[] | select(.name != null) | select(.name | contains("mock.istio-verify-ext"))] | length'

echo "--- CMD7: client upstream_rq_retry counters for mock (expect none: retries silently ignored) ---"
kubectl -n "${NS}" exec client -c istio-proxy -- curl -s localhost:15000/stats | grep 'upstream_rq_retry' | grep mock \
  || echo 'no retry counters exist for this host (route never compiled)'

echo "--- EXTRA: root-cause check -- CDS/EDS for hop-gw (should now be healthy, unlike attempt1) ---"
istioctl proxy-config cluster "client.${NS}" -o json | jq -r '.[] | select(.name != null) | select(.name | contains("hop-gw")) | .name'
istioctl proxy-config endpoints "client.${NS}" | grep -i hop-gw || echo "NO ENDPOINTS FOUND for hop-gw"

echo "--- EXTRA: root-cause check -- mock's dedicated per-ClusterIP listener bypasses SNI/gateway routing ---"
MOCK_IP="$(kubectl get svc mock -n istio-verify-ext -o jsonpath='{.spec.clusterIP}')"
istioctl proxy-config listener "client.${NS}" --port 443 -o json \
  | jq --arg name "${MOCK_IP}_443" -r '.[] | select(.name==$name) | .filterChains[] | {serverNames:(.filterChainMatch.serverNames//["<default>"]), filters:[.filters[].name], cluster:[.filters[].typedConfig.cluster]}'

echo "--- EXTRA: behavioral proof -- scale hop-gw to 0, request should still succeed if bypassed ---"
kubectl -n "${NS}" scale deploy/hop-gw --replicas=0
kubectl -n "${NS}" wait --for=delete pod -l istio=hop-gw --timeout=60s
kubectl -n "${NS}" exec client -c curl -- curl -sk -o /dev/null -w 'twohop_https_gwdown=%{http_code}\n' \
  https://mock.istio-verify-ext.svc.homelab.local/
kubectl -n "${NS}" scale deploy/hop-gw --replicas=1
kubectl -n "${NS}" rollout status deploy/hop-gw --timeout=90s

echo "--- EXTRA: sanity -- client -> echo in-mesh HTTP should still show normal L7 access log ---"
kubectl -n "${NS}" exec client -c curl -- curl -s -o /dev/null -w 'echo_http=%{http_code}\n' http://echo/
kubectl -n "${NS}" logs deploy/echo -c istio-proxy --since=1m | tail -3

echo "--- done ---"