云平台控制面单点依赖的代价：Railway 八小时全平台宕机复盘

┌─────────────────────────────────┐
│         GCP (控制面)              │
│  API / Scheduler / DB / Router  │
└──────────────┬──────────────────┘
               │ 单一依赖点
       ┌───────┴────────┐
       │                │
┌──────┴──────┐  ┌──────┴──────┐  ┌──────────┐
│  AWS 数据面  │  │ GCP 数据面  │  │ 裸金属    │
│  (工作负载)  │  │ (工作负载)  │  │ (工作负载) │
└─────────────┘  └─────────────┘  └──────────┘

# control-plane-deployment.yaml
# 在 Provider-A 和 Provider-B 的集群上各部署一份
apiVersion: apps/v1
kind: Deployment
metadata:
  name: railway-api-server
  namespace: control-plane
spec:
  replicas: 3
  selector:
    matchLabels:
      app: railway-api
  template:
    metadata:
      labels:
        app: railway-api
        cluster: multi-active  # 标记为多活控制面
    spec:
      containers:
      - name: api
        image: railway/api-server:v2.1.0
        ports:
        - containerPort: 8080
        env:
        - name: CLUSTER_ID
          valueFrom:
            fieldRef:
              fieldPath: metadata.name  # 区分不同提供商上的实例
        - name: DB_ENDPOINT
          value: "db-primary.control-plane.internal"  # 跨云可达的数据库
        livenessProbe:
          httpGet:
            path: /healthz
            port: 8080
          initialDelaySeconds: 10
          periodSeconds: 5
        readinessProbe:
          httpGet:
            path: /readyz
            port: 8080
          initialDelaySeconds: 5
          periodSeconds: 3

# 用 Cloudflare API 设置权重路由
# Provider-A 权重 70%，Provider-B 权重 30%（日常流量分配）

# 主记录
curl -X POST "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records" \
  -H "Authorization: Bearer ${CF_API_TOKEN}" \
  -H "Content-Type: application/json" \
  --data '{
    "type": "CNAME",
    "name": "api.railway.app",
    "content": "cp-a.railway-infra.com",
    "proxied": true,
    "priority": 10,
    "weight": 70
  }'

# 备记录
curl -X POST "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records" \
  -H "Authorization: Bearer ${CF_API_TOKEN}" \
  -H "Content-Type: application/json" \
  --data '{
    "type": "CNAME",
    "name": "api.railway.app",
    "content": "cp-b.railway-infra.com",
    "proxied": true,
    "priority": 10,
    "weight": 30
  }'

#!/usr/bin/env bash
# failover-watchdog.sh — 每分钟检查控制面健康，自动调整 DNS 权重
# 部署在独立于两个提供商的第三位置（如裸金属或边缘节点）

set -euo pipefail

CF_API_TOKEN="${CF_API_TOKEN:?请设置 Cloudflare API Token}"
ZONE_ID="${ZONE_ID:?请设置 Zone ID}"
PRIMARY_ENDPOINT="https://cp-a.railway-infra.com/healthz"
FAILBACK_ENDPOINT="https://cp-b.railway-infra.com/healthz"
LOG_TAG="[failover-watchdog]"

log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) ${LOG_TAG} $*"; }

check_health() {
  local url="$1"
  curl -sf -m 5 "$url" >/dev/null 2>&1
}

current_state="healthy"  # 初始状态

while true; do
  primary_ok=$(check_health "$PRIMARY_ENDPOINT" && echo "yes" || echo "no")
  failback_ok=$(check_health "$FAILBACK_ENDPOINT" && echo "yes" || echo "no")

  if [[ "$primary_ok" == "no" && "$current_state" == "healthy" ]]; then
    log "主控制面不可用，切换流量到备用"
    # 调用 Cloudflare API 将权重改为 primary=0, failback=100
    # （此处省略具体 API 调用，逻辑同第二步，仅改 weight 值）
    current_state="failback"
  elif [[ "$primary_ok" == "yes" && "$current_state" == "failback" ]]; then
    log "主控制面恢复，逐步回切流量"
    # 先设 primary=30, failback=70，观察 5 分钟后再完全回切
    current_state="recovering"
  elif [[ "$primary_ok" == "yes" && "$current_state" == "recovering" ]]; then
    log "回切稳定，恢复日常权重 70/30"
    current_state="healthy"
  elif [[ "$primary_ok" == "no" && "$failback_ok" == "no" ]]; then
    log "⚠️ 双控制面均不可用，触发全平台告警"
    # 发送 PagerDuty / Slack 告警
  fi

  sleep 60
done

# 在 Provider-A 的主库上配置逻辑复制发布
psql -U postgres -d railway_control -c "
  CREATE PUBLICATION control_plane_pub FOR ALL TABLES;
"

# 在 Provider-B 的备库上配置订阅
psql -U postgres -d railway_control -c "
  CREATE SUBSCRIPTION control_plane_sub
  CONNECTION 'host=pg-primary.cp-a.internal port=5432 dbname=railway_control user=replicator password=REPLICATOR_PASSWORD'
  PUBLICATION control_plane_pub;
"