数据中心断电零预警：Meta 的"瞬时断电风暴"验证体系

# 给节点打上 power-domain 标签
kubectl label node node-1 node-2 power-domain=domain-a
kubectl label node node-3 node-4 power-domain=domain-b
kubectl label node node-5 power-domain=domain-c

apiVersion: chaos-mesh.org/v1alpha1
kind: PodChaos
metadata:
  name: powerloss-domain-a-pods
  namespace: chaos-testing
spec:
  action: pod-kill
  mode: all
  selector:
    namespaces:
      - production
    labelSelectors:
      app: my-critical-service
    nodeSelectors:
      power-domain: domain-a
  gracePeriod: 0          # 关键：零优雅期，模拟瞬间死亡
  duration: "0s"          # pod-kill 不需要 duration，立即生效
---
apiVersion: chaos-mesh.org/v1alpha1
kind: NetworkChaos
metadata:
  name: powerloss-domain-a-network
  namespace: chaos-testing
spec:
  action: partition
  direction: both
  mode: all
  selector:
    namespaces:
      - production
    labelSelectors:
      app: my-critical-service
    nodeSelectors:
      power-domain: domain-a
  duration: "300s"        # 断网持续 5 分钟，模拟断电后恢复前的时间窗口

#!/usr/bash
# powerloss-readiness-check.sh
# 在混沌注入前后运行，验证服务容错能力

set -euo pipefail

SERVICE_URL="https://my-critical-service.example.com/api/v1/health"
EXPECTED_AVAILABILITY_PCT=95    # 断电期间剩余集群可用率不低于 95%
MAX_DATA_LOSS_COUNT=0           # 数据丢失容忍量

echo "=== Pre-Flight Check ==="
echo "Checking service baseline availability..."
pre_status=$(curl -s -o /dev/null -w "%{http_code}" "$SERVICE_URL")
echo "Baseline status: $pre_status"

echo ""
echo "=== Injecting Chaos: domain-a power loss ==="
echo "Run: kubectl apply -f powerloss-domain-a.yaml"
echo "Waiting 10s for chaos to take effect..."
sleep 10

echo ""
echo "=== During-Fault Measurement ==="
success_count=0
total_count=0
for i in $(seq 1 30); do
  code=$(curl -s -o /dev/null -w "%{http_code}" "$SERVICE_URL" || echo "000")
  total_count=$((total_count + 1))
  if [[ "$code" =~ ^2 ]]; then
    success_count=$((success_count + 1))
  fi
  echo "  Request $i: HTTP $code"
  sleep 1
done

availability=$((success_count * 100 / total_count))
echo "Availability during fault: ${availability}% (target: ${EXPECTED_AVAILABILITY_PCT}%)"

echo ""
echo "=== Recovery Check ==="
echo "Run: kubectl delete -f powerloss-domain-a.yaml"
echo "Waiting 120s for recovery..."
sleep 120

post_status=$(curl -s -o /dev/null -w "%{http_code}" "$SERVICE_URL")
echo "Post-recovery status: $post_status"

echo ""
echo "=== Result ==="
if [[ $availability -ge $EXPECTED_AVAILABILITY_PCT ]] && [[ "$post_status" =~ ^2 ]]; then
  echo "PASS: Service met availability target during fault and recovered."
else
  echo "FAIL: Service did not meet target. Investigate before proceeding to larger fault domains."
  exit 1
fi

# 1. 先部署 Chaos Mesh 到集群
kubectl apply -f powerloss-domain-a.yaml

# 2. 执行验证脚本
bash powerloss-readiness-check.sh

# 3. 清理混沌实验
kubectl delete -f powerloss-domain-a.yaml