支付系统里的 ECS 混沌工程：别让故障注入变成真实事故

#!/usr/bin/env bash
set -euo pipefail

: "${CLUSTER:?Set CLUSTER}"
: "${SERVICE:?Set SERVICE}"
: "${ALARM_NAMES:?Set comma-separated ALARM_NAMES}"

AWS_REGION="${AWS_REGION:-us-east-1}"
ENVIRONMENT="${ENVIRONMENT:-unset}"
RUN_CHAOS="${RUN_CHAOS:-false}"

if [[ "$ENVIRONMENT" != "staging" ]]; then
  echo "Refusing to run outside staging" >&2
  exit 1
fi

service_json=$(aws ecs describe-services \
  --region "$AWS_REGION" \
  --cluster "$CLUSTER" \
  --services "$SERVICE" \
  --output json)

desired=$(jq -r '.services[0].desiredCount' <<<"$service_json")
running=$(jq -r '.services[0].runningCount' <<<"$service_json")
deployments=$(jq -r '.services[0].deployments | length' <<<"$service_json")
rollout=$(jq -r '.services[0].deployments[0].rolloutState // "UNKNOWN"' <<<"$service_json")

if (( desired < 2 )) || (( running != desired )); then
  echo "Service lacks safe headroom: desired=$desired running=$running" >&2
  exit 1
fi

if (( deployments != 1 )) || [[ "$rollout" != "COMPLETED" ]]; then
  echo "Service is not in a stable deployment state" >&2
  exit 1
fi

IFS=',' read -ra alarms <<<"$ALARM_NAMES"
for alarm in "${alarms[@]}"; do
  state=$(aws cloudwatch describe-alarms \
    --region "$AWS_REGION" \
    --alarm-names "$alarm" \
    --query 'MetricAlarms[0].StateValue' \
    --output text)

  if [[ "$state" != "OK" ]]; then
    echo "Alarm $alarm is $state; experiment blocked" >&2
    exit 1
  fi
done

task=$(aws ecs list-tasks \
  --region "$AWS_REGION" \
  --cluster "$CLUSTER" \
  --service-name "$SERVICE" \
  --desired-status RUNNING \
  --query 'taskArns[0]' \
  --output text)

if [[ -z "$task" || "$task" == "None" ]]; then
  echo "No running task found" >&2
  exit 1
fi

echo "Selected task: $task"

if [[ "$RUN_CHAOS" != "true" ]]; then
  echo "Dry run only. Set RUN_CHAOS=true to stop this staging task."
  exit 0
fi

aws ecs stop-task \
  --region "$AWS_REGION" \
  --cluster "$CLUSTER" \
  --task "$task" \
  --reason 'bounded staging chaos experiment' \
  >/dev/null

echo "Task stopped. Watch business SLOs, retry load, DNS timing, and AZ distribution."

CLUSTER=payments-staging \
SERVICE=payment-api \
AWS_REGION=ap-southeast-1 \
ENVIRONMENT=staging \
ALARM_NAMES=PaymentErrorRate,DatabaseConnections \
./ecs-chaos.sh