#!/usr/bin/env bash
# T29: SDS UDS emptyDir 볼륨(workload-socket/credential-socket/workload-certs)을
# 제거한 injected pod이 실제로 CrashLoopBackOff에 빠지는지, 로그에
# 'SDS grpc server ... failed to set up UDS: ... bind: no such file or directory'
# 에러가 찍히는지 직접 관찰한다.
#
# NOTE: Kubernetes Pod의 volumeMounts/volumes는 불변(immutable) 필드이므로
# 이미 떠 있는 injected pod을 in-place로 patch할 수 없다. 따라서:
#   1) 정상 주입된 pod을 한 번 띄워 istio-proxy의 실제 volumeMounts/volumes를 캡처
#   2) 그 스펙에서 workload-socket/credential-socket/workload-certs 3종만 제거
#   3) sidecar.istio.io/inject=false 로 재적용 (라벨을 true로 유지하면 네임스페이스
#      auto-injection webhook이 새 Pod 생성 이벤트로 인식해 제거한 볼륨을 도로
#      주입해 원복시켜 버림 -- 본 테스트에서 실측 확인된 사실)
set -euo pipefail

NS=istio-vt-t29
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
MANIFEST="$SCRIPT_DIR/manifest.yaml"
WORKDIR="$(mktemp -d)"
LIVE_YAML="$WORKDIR/sds-broken-live.yaml"
STRIPPED="$WORKDIR/sds-broken-stripped.yaml"

cleanup() {
  echo "+ kubectl delete namespace $NS --wait=false --ignore-not-found"
  kubectl delete namespace "$NS" --wait=false --ignore-not-found
  rm -rf "$WORKDIR"
}
trap cleanup EXIT

echo "+ kubectl create namespace $NS"
kubectl create namespace "$NS"
echo "+ kubectl label namespace $NS istio-injection=enabled"
kubectl label namespace "$NS" istio-injection=enabled

echo "+ kubectl apply -f manifest.yaml (baseline: normal sidecar injection)"
kubectl apply -f "$MANIFEST"

echo "+ kubectl -n $NS wait --for=condition=Ready pod/sds-broken --timeout=120s"
kubectl -n "$NS" wait --for=condition=Ready pod/sds-broken --timeout=120s

echo "+ kubectl -n $NS get pod sds-broken (expect 2/2 Running)"
kubectl -n "$NS" get pod sds-broken

echo "+ capture injected pod spec to confirm the 3 target volumeMounts/volumes exist"
kubectl -n "$NS" get pod sds-broken -o yaml > "$LIVE_YAML"
grep -E 'workload-socket|credential-socket|workload-certs' "$LIVE_YAML" || true

echo "+ dynamically build a stripped pod spec: remove workload-socket/credential-socket/workload-certs"
echo "  volumeMounts+volumes from the captured injected spec; force sidecar.istio.io/inject=false so the"
echo "  namespace auto-injection webhook does NOT re-inject/restore them on the recreate (kubectl apply of an"
echo "  already-injected pod with inject=true triggers a fresh webhook mutation on the new Pod CREATE event and"
echo "  silently restores the removed volumes -- confirmed empirically in this test)."
python3 - "$LIVE_YAML" "$STRIPPED" << 'PYEOF'
import sys, yaml

live_path, out_path = sys.argv[1], sys.argv[2]
with open(live_path) as f:
    d = yaml.safe_load(f)

STRIP = {"workload-socket", "credential-socket", "workload-certs"}
spec = d["spec"]

def strip_mounts(containers):
    out = []
    for c in containers:
        c = dict(c)
        if "volumeMounts" in c:
            c["volumeMounts"] = [vm for vm in c["volumeMounts"] if vm["name"] not in STRIP]
        out.append(c)
    return out

new_pod = {
    "apiVersion": "v1",
    "kind": "Pod",
    "metadata": {
        "name": d["metadata"]["name"],
        "namespace": d["metadata"]["namespace"],
        "labels": {**d["metadata"].get("labels", {}), "sidecar.istio.io/inject": "false"},
        "annotations": {**d["metadata"].get("annotations", {}), "sidecar.istio.io/inject": "false"},
    },
    "spec": {
        "serviceAccountName": spec.get("serviceAccountName", "default"),
        "restartPolicy": spec.get("restartPolicy", "Always"),
        "initContainers": strip_mounts(spec.get("initContainers", [])),
        "containers": strip_mounts(spec["containers"]),
        "volumes": [v for v in spec["volumes"] if v["name"] not in STRIP],
    },
}
with open(out_path, "w") as f:
    yaml.dump(new_pod, f, default_flow_style=False, sort_keys=False)
PYEOF
echo "  -> wrote $STRIPPED"

echo "+ delete pod, reapply stripped manifest (istio-proxy volumeMounts for the 3 SDS UDS dirs + matching volumes removed; sidecar.istio.io/inject=false to stop the webhook from re-injecting/restoring them)"
kubectl delete pod sds-broken -n "$NS" --wait=true
kubectl apply -f "$STRIPPED"

echo "+ sleep 20 (let istio-proxy attempt SDS UDS bind)"
sleep 20

echo "+ kubectl -n $NS get pod sds-broken"
kubectl -n "$NS" get pod sds-broken

echo "+ ISTIO_PROXY_STATE"
kubectl -n "$NS" get pod sds-broken -o jsonpath='{.status.containerStatuses[?(@.name=="istio-proxy")].state}'
echo

echo "+ current istio-proxy log grep for the SDS UDS bind error (expected to appear immediately)"
kubectl -n "$NS" logs sds-broken -c istio-proxy 2>/dev/null | grep -i 'failed to set up UDS' | tail -5 || true

# NOTE (empirically observed in this test): istio-proxy has NO livenessProbe, only a
# readinessProbe and a startupProbe (failureThreshold=600, periodSeconds=1s). The SDS UDS
# bind failure alone does NOT crash the process -- Envoy/pilot-agent stays alive, logging the
# error on every retry, while readiness never succeeds. The container is only killed+restarted
# once the startupProbe has failed ~600 consecutive times (~10 minutes), and it restarts with
# ~zero backoff (exit reason=Completed, exitCode=0) -- i.e. pod STATUS shows "Running", not
# "CrashLoopBackOff", for the entire time; only restartCount ticks up roughly once per ~10min.
# Poll for up to ~11 minutes to actually observe that first probe-driven restart.
echo "+ polling up to 660s for istio-proxy restartCount to increase (startupProbe failureThreshold=600 * periodSeconds=1s)"
END=$((SECONDS+660))
while [ $SECONDS -lt $END ]; do
  restart=$(kubectl -n "$NS" get pod sds-broken -o jsonpath='{.status.containerStatuses[?(@.name=="istio-proxy")].restartCount}' 2>/dev/null || echo 0)
  echo "  [t=${SECONDS}s] restartCount=$restart"
  if [ "$restart" -ge 1 ]; then
    echo "  -> restarted"
    break
  fi
  sleep 20
done

echo "+ kubectl -n $NS get pod sds-broken (final)"
kubectl -n "$NS" get pod sds-broken

echo "+ restartCount"
kubectl -n "$NS" get pod sds-broken -o jsonpath='{.status.containerStatuses[?(@.name=="istio-proxy")].restartCount}{"\n"}'

echo "+ (if restarted at least once) previous istio-proxy log grep for the SDS UDS bind error"
kubectl -n "$NS" logs sds-broken -c istio-proxy --previous 2>/dev/null | grep -i 'failed to set up UDS' | tail -5 || true

echo "done."