Disaster Recovery on AWS

  # Automated RDS backups (enable at creation)
aws rds create-db-instance \
    --backup-retention-period 35 \
    --preferred-backup-window "03:00-04:00" \
    ...

# Cross-region automated backup copy
aws rds start-db-instance-automated-backups-replication \
    --source-db-instance-arn arn:aws:rds:us-east-1:123:db:myapp-postgres \
    --backup-retention-period 35 \
    --kms-key-id arn:aws:kms:us-west-2:123:key/xxx

# Manual snapshot before major changes
aws rds create-db-snapshot \
    --db-instance-identifier myapp-postgres \
    --db-snapshot-identifier pre-migration-$(date +%Y%m%d)

# Restore to new instance in DR region
aws rds restore-db-instance-from-db-snapshot \
    --db-instance-identifier myapp-postgres-dr \
    --db-snapshot-arn arn:aws:rds:us-west-2:123:snapshot:myapp-postgres-auto-xxx \
    --db-instance-class db.r6g.large \
    --vpc-security-group-ids sg-dr-database \
    --db-subnet-group-name dr-db-subnet-group
  

  # Enable versioning (required for replication)
aws s3api put-bucket-versioning \
    --bucket myapp-assets \
    --versioning-configuration Status=Enabled

# Create replication rule
aws s3api put-bucket-replication \
    --bucket myapp-assets \
    --replication-configuration '{
        "Role": "arn:aws:iam::123:role/s3-replication-role",
        "Rules": [{
            "Status": "Enabled",
            "Priority": 1,
            "Filter": {},
            "Destination": {
                "Bucket": "arn:aws:s3:::myapp-assets-dr",
                "ReplicationTime": {"Status": "Enabled", "Time": {"Minutes": 15}},
                "Metrics": {"Status": "Enabled", "EventThreshold": {"Minutes": 15}}
            },
            "DeleteMarkerReplication": {"Status": "Enabled"}
        }]
    }'
  

  # Promote RDS read replica to standalone (failover)
aws rds promote-read-replica \
    --db-instance-identifier myapp-postgres-dr-replica

# Scale ECS service from 0 to production capacity
aws ecs update-service \
    --cluster dr-production \
    --service myapp-service \
    --desired-count 4
  

  # DR region runs at 20-30% capacity
# Primary: 10 EC2 instances → DR: 2-3 instances always running

# On failover, scale DR ASG to full capacity
aws autoscaling update-auto-scaling-group \
    --auto-scaling-group-name dr-web-asg \
    --desired-capacity 10 \
    --min-size 10
  

  # Route 53 latency-based routing
aws route53 change-resource-record-sets \
    --hosted-zone-id Z1234567890 \
    --change-batch '{
        "Changes": [{
            "Action": "UPSERT",
            "ResourceRecordSet": {
                "Name": "api.example.com",
                "Type": "A",
                "SetIdentifier": "us-east-1",
                "Region": "us-east-1",
                "AliasTarget": {
                    "HostedZoneId": "Z35SXDOTRQ7X7K",
                    "DNSName": "primary-alb.us-east-1.elb.amazonaws.com",
                    "EvaluateTargetHealth": true
                }
            }
        }]
    }'
  

  aws rds create-global-cluster \
    --global-cluster-identifier myapp-global \
    --engine aurora-postgresql \
    --engine-version 16.1

aws rds create-db-cluster \
    --db-cluster-identifier myapp-primary \
    --engine aurora-postgresql \
    --global-cluster-identifier myapp-global

aws rds create-db-cluster \
    --db-cluster-identifier myapp-dr \
    --engine aurora-postgresql \
    --global-cluster-identifier myapp-global \
    --region us-west-2

# Failover to DR region (< 1 minute)
aws rds failover-global-cluster \
    --global-cluster-identifier myapp-global \
    --target-db-cluster-identifier myapp-dr
  

  aws dynamodb create-table \
    --table-name Orders \
    --attribute-definitions AttributeName=orderId,AttributeType=S \
    --key-schema AttributeName=orderId,KeyType=HASH \
    --billing-mode PAY_PER_REQUEST \
    --stream-specification StreamEnabled=true,StreamViewType=NEW_AND_OLD_IMAGES

aws dynamodb create-global-table \
    --global-table-name Orders \
    --replication-group RegionName=us-east-1 RegionName=eu-west-1 RegionName=ap-southeast-1
  

  # DR Failover Runbook: Production API

## Trigger Conditions
- Primary region (us-east-1) unavailable for > 5 minutes
- RDS primary failure with Multi-AZ failover unsuccessful
- Security incident requiring region isolation

## Failover Steps
1. Confirm primary region outage (CloudWatch, Route 53 health checks)
2. Notify stakeholders via PagerDuty (#incident-dr channel)
3. Promote RDS read replica in us-west-2 (Step 3.1 below)
4. Scale ECS services in DR region to production capacity
5. Update Route 53 failover record to point to DR ALB
6. Verify application health: curl https://api.example.com/health
7. Monitor DR region metrics for 30 minutes
8. Document incident timeline for postmortem

## Rollback Steps
1. Confirm primary region restored and stable
2. Sync data from DR back to primary (if applicable)
3. Scale DR back to standby capacity
4. Update Route 53 to primary region
5. Verify and notify stakeholders

## Test Schedule
- Quarterly: Full failover drill (non-business hours)
- Monthly: Backup restore verification
- Weekly: Automated DR health check script
  

  # Create backup vault
aws backup create-backup-vault --backup-vault-name production-vault

# Backup plan: daily with 35-day retention
aws backup create-backup-plan --backup-plan '{
    "BackupPlanName": "daily-production",
    "Rules": [{
        "RuleName": "daily-backup",
        "TargetBackupVaultName": "production-vault",
        "ScheduleExpression": "cron(0 5 ? * * *)",
        "StartWindowMinutes": 60,
        "CompletionWindowMinutes": 120,
        "Lifecycle": {"DeleteAfterDays": 35},
        "CopyActions": [{
            "Lifecycle": {"DeleteAfterDays": 35},
            "DestinationBackupVaultArn": "arn:aws:backup:us-west-2:123:backup-vault:dr-vault"
        }]
    }]
}'