灾难恢复不是买工具，而是把恢复练成肌肉记忆

#!/usr/bin/env bash
set -euo pipefail

# Assumptions to change before running:
# - STANDBY_HOST points to the replica you selected before the incident.
# - PGDATA matches the PostgreSQL data directory on that host.
# - APP_HEALTHCHECK is an application endpoint reachable from this machine.
# - You have SSH and sudo permissions for the standby.

STANDBY_HOST="postgres-standby-1.example.internal"
PGDATA="/var/lib/postgresql/16/main"
APP_HEALTHCHECK="https://app.example.com/healthz"
MAX_REPLAY_LAG_BYTES="16777216" # 16 MiB example threshold

log() {
  printf '[%s] %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"
}

log "Checking standby replay lag"
LAG_BYTES=$(ssh "$STANDBY_HOST" "sudo -u postgres psql -tAc \"SELECT COALESCE(pg_wal_lsn_diff(pg_last_wal_receive_lsn(), pg_last_wal_replay_lsn()), 0)::bigint;\"" | tr -d ' ')

log "Replay lag bytes: ${LAG_BYTES}"
if [ "$LAG_BYTES" -gt "$MAX_REPLAY_LAG_BYTES" ]; then
  log "STOP: replay lag is above threshold. Escalate to incident commander for risk authorization."
  exit 2
fi

log "Promoting standby"
ssh "$STANDBY_HOST" "sudo -u postgres /usr/lib/postgresql/16/bin/pg_ctl promote -D '$PGDATA'"

log "Verifying the promoted node is writable"
ssh "$STANDBY_HOST" "sudo -u postgres psql -tAc \"CREATE TABLE IF NOT EXISTS dr_validation(ts timestamptz DEFAULT now()); INSERT INTO dr_validation DEFAULT VALUES; SELECT count(*) FROM dr_validation;\""

log "At this point, update DNS/load balancer according to your environment."
log "Waiting 20 seconds before app health check"
sleep 20

HTTP_CODE=$(curl -sS -o /dev/null -w '%{http_code}' "$APP_HEALTHCHECK")
if [ "$HTTP_CODE" != "200" ]; then
  log "FAIL: application health check returned HTTP ${HTTP_CODE}. Keep traffic disabled and escalate."
  exit 3
fi

log "SUCCESS: promoted standby is writable and application health check passed."

apiVersion: v1
kind: Namespace
metadata:
  name: dr-gameday
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: dr-drill-plan
  namespace: dr-gameday
data:
  objective: "Restore service within 30 minutes after simulated primary database loss"
  failure_mode: "Primary PostgreSQL instance unavailable"
  success_criteria: |
    1. Standby promoted successfully
    2. Application health endpoint returns HTTP 200
    3. Incident update sent every 15 minutes
    4. Actual RTO and decision time recorded
  rollback: |
    Stop application writes, keep promoted standby isolated, and escalate to incident commander.