# 14 · Disaster Recovery for ~$5/Month (Pilot Light)

**Format:** AWS CDK (TypeScript), 2 stacks in 2 regions + Step Functions drill · **Est. cost:** ~$1–5/month (backup storage + a few minutes of a restored t4g.nano per drill)

Most DR plans are a PDF nobody has tested. This one **tests itself every month and emails you the measured RTO/RPO**.

```
us-east-1 (primary)                               us-west-2 (DR, "pilot light")
────────────────────                              ───────────────────────────────
resources tagged dr=true                          dr-vault  ◀── cross-region copy (daily, 14-day retention)
   └─ AWS Backup plan (daily 05:00 UTC) ───────▶  empty VPC + SG (costs $0 until used)
                                                  Step Functions "dr-restore-drill":
                                                    find latest point → restore EC2 → wait →
                                                    status + HTTP checks → measure RTO → terminate → report
                                                  EventBridge Scheduler: first Monday of each month
```

## Deploy
```bash
npm install
npx cdk bootstrap aws://ACCOUNT/us-east-1 aws://ACCOUNT/us-west-2
npx cdk deploy --all -c alertEmail=you@example.com
# custom regions: -c primaryRegion=us-east-2 -c drRegion=us-west-1
```
Protect your own resources by tagging them `dr=true` (EC2, EBS, RDS, DynamoDB, EFS…). The demo stack includes a tagged nginx instance.

## Run a drill now
Wait for the first nightly backup and copy (or start an on-demand backup and copy job), then:
```bash
aws stepfunctions start-execution --region us-west-2 \
  --state-machine-arn <StateMachineArn> --input '{"resource_type":"EC2","cleanup":true}'
```
You'll get an email like:
```json
{ "result": "PASS", "measured_rto_minutes": 11.4, "rpo_hours_at_drill_time": 9.2, "cleaned_up": true }
```

## Real failover
Same state machine with `"cleanup": false`. Then point DNS (Route 53 failover record) at the restored instance.

## Upgrade paths
| Need | Use |
|---|---|
| RPO in seconds for servers | AWS Elastic Disaster Recovery (continuous block replication) |
| Database RPO < 1 s | Aurora Global Database / DynamoDB global tables (see project 04) |
| Ransomware-proof copies | Uncomment `lockConfiguration` (vault lock) + a separate DR account |

## Clean up
```bash
npx cdk destroy --all   # delete recovery points in both vaults first if destroy fails
```

#!/usr/bin/env node
import * as cdk from 'aws-cdk-lib';
import { DrRegionStack, DR_VAULT_NAME } from '../lib/dr-region-stack';
import { PrimaryRegionStack } from '../lib/primary-region-stack';

const app = new cdk.App();
const account = process.env.CDK_DEFAULT_ACCOUNT;
const primaryRegion: string = app.node.tryGetContext('primaryRegion') ?? 'us-east-1';
const drRegion: string = app.node.tryGetContext('drRegion') ?? 'us-west-2';
const alertEmail: string | undefined = app.node.tryGetContext('alertEmail');

const dr = new DrRegionStack(app, 'PilotLight-DR', {
  env: { account, region: drRegion },
  alertEmail,
});

const primary = new PrimaryRegionStack(app, 'PilotLight-Primary', {
  env: { account, region: primaryRegion },
  drRegion,
  drVaultName: DR_VAULT_NAME,
  withDemoInstance: app.node.tryGetContext('withDemoInstance') !== 'false',
});
primary.addStackDependency(dr); // the copy target vault must exist first

import * as path from 'path';
import * as cdk from 'aws-cdk-lib';
import { Construct } from 'constructs';
import * as backup from 'aws-cdk-lib/aws-backup';
import * as ec2 from 'aws-cdk-lib/aws-ec2';
import * as iam from 'aws-cdk-lib/aws-iam';
import * as lambda from 'aws-cdk-lib/aws-lambda';
import * as logs from 'aws-cdk-lib/aws-logs';
import * as scheduler from 'aws-cdk-lib/aws-scheduler';
import * as sns from 'aws-cdk-lib/aws-sns';
import * as subs from 'aws-cdk-lib/aws-sns-subscriptions';
import * as sfn from 'aws-cdk-lib/aws-stepfunctions';
import * as tasks from 'aws-cdk-lib/aws-stepfunctions-tasks';

export const DR_VAULT_NAME = 'dr-vault';

export interface DrRegionStackProps extends cdk.StackProps {
  alertEmail?: string;
}

/**
 * The "pilot light": an empty-but-ready DR network, a vault receiving copies, and a
 * one-click (and monthly scheduled) restore drill that measures real RTO.
 */
export class DrRegionStack extends cdk.Stack {
  constructor(scope: Construct, id: string, props: DrRegionStackProps) {
    super(scope, id, props);

    const vault = new backup.BackupVault(this, 'DrVault', {
      backupVaultName: DR_VAULT_NAME,
      removalPolicy: cdk.RemovalPolicy.DESTROY,
      // Uncomment for ransomware protection (WORM). Compliance mode cannot be undone!
      // lockConfiguration: { minRetention: cdk.Duration.days(7) },
    });

    // Pilot light network: costs $0 until something runs in it.
    const vpc = new ec2.Vpc(this, 'DrVpc', {
      maxAzs: 2,
      natGateways: 0,
      subnetConfiguration: [{ name: 'public', subnetType: ec2.SubnetType.PUBLIC }],
    });
    const sg = new ec2.SecurityGroup(this, 'RestoredSg', { vpc, description: 'Restored workloads' });
    sg.addIngressRule(ec2.Peer.anyIpv4(), ec2.Port.tcp(80), 'HTTP for validation');

    const restoreRole = new iam.Role(this, 'RestoreRole', {
      assumedBy: new iam.ServicePrincipal('backup.amazonaws.com'),
      managedPolicies: [iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSBackupServiceRolePolicyForRestores')],
    });
    restoreRole.addToPolicy(new iam.PolicyStatement({ actions: ['iam:PassRole'], resources: ['*'] }));

    const topic = new sns.Topic(this, 'DrillReports', { displayName: 'DR drill reports' });
    if (props.alertEmail) topic.addSubscription(new subs.EmailSubscription(props.alertEmail));

    const drillFn = new lambda.Function(this, 'DrillFn', {
      runtime: lambda.Runtime.PYTHON_3_12,
      architecture: lambda.Architecture.ARM_64,
      handler: 'drill.handler',
      code: lambda.Code.fromAsset(path.join(__dirname, '..', 'lambda')),
      timeout: cdk.Duration.minutes(1),
      environment: {
        VAULT_NAME: vault.backupVaultName,
        SUBNET_ID: vpc.publicSubnets[0].subnetId,
        SECURITY_GROUP_ID: sg.securityGroupId,
        RESTORE_ROLE_ARN: restoreRole.roleArn,
        TOPIC_ARN: topic.topicArn,
      },
      logGroup: new logs.LogGroup(this, 'DrillLogs', { retention: logs.RetentionDays.THREE_MONTHS, removalPolicy: cdk.RemovalPolicy.DESTROY }),
    });
    topic.grantPublish(drillFn);
    drillFn.addToRolePolicy(
      new iam.PolicyStatement({
        actions: [
          'backup:ListRecoveryPointsByBackupVault',
          'backup:GetRecoveryPointRestoreMetadata',
          'backup:StartRestoreJob',
          'backup:DescribeRestoreJob',
          'ec2:DescribeInstances',
          'ec2:DescribeInstanceStatus',
          'ec2:TerminateInstances',
          'ec2:CreateTags',
          'cloudwatch:PutMetricData',
        ],
        resources: ['*'],
      }),
    );
    drillFn.addToRolePolicy(new iam.PolicyStatement({ actions: ['iam:PassRole'], resources: [restoreRole.roleArn] }));

    // ---------------------------------------------------------------- the drill as a state machine
    const step = (name: string, action: string) =>
      new tasks.LambdaInvoke(this, name, {
        lambdaFunction: drillFn,
        payload: sfn.TaskInput.fromObject({ action, 'state.$': '$' }),
        payloadResponseOnly: true,
        retryOnServiceExceptions: true,
      });

    const findLatest = step('FindLatestRecoveryPoint', 'find_latest');
    const startRestore = step('StartRestore', 'start_restore');
    const checkRestore = step('CheckRestore', 'check_restore');
    const validate = step('ValidateWorkload', 'validate');
    const cleanup = step('CleanupAndReport', 'cleanup_and_report');
    const waitRestore = new sfn.Wait(this, 'WaitForRestore', { time: sfn.WaitTime.duration(cdk.Duration.seconds(60)) });
    const waitHealthy = new sfn.Wait(this, 'WaitForHealthy', { time: sfn.WaitTime.duration(cdk.Duration.seconds(30)) });
    const failed = new sfn.Fail(this, 'DrillFailed', { error: 'RestoreFailed' });

    const healthyChoice = new sfn.Choice(this, 'Healthy?')
      .when(sfn.Condition.booleanEquals('$.healthy', true), cleanup)
      .when(sfn.Condition.numberGreaterThan('$.health_checks', 20), cleanup)
      .otherwise(waitHealthy.next(validate));

    const restoreChoice = new sfn.Choice(this, 'RestoreDone?')
      .when(sfn.Condition.stringEquals('$.restore_status', 'COMPLETED'), validate.next(healthyChoice))
      .when(sfn.Condition.or(
        sfn.Condition.stringEquals('$.restore_status', 'FAILED'),
        sfn.Condition.stringEquals('$.restore_status', 'ABORTED'),
      ), failed)
      .otherwise(waitRestore.next(checkRestore));

    const machine = new sfn.StateMachine(this, 'RestoreDrill', {
      stateMachineName: 'dr-restore-drill',
      timeout: cdk.Duration.hours(2),
      definitionBody: sfn.DefinitionBody.fromChainable(findLatest.next(startRestore).next(checkRestore).next(restoreChoice)),
      tracingEnabled: true,
    });

    // Monthly unannounced drill (first Monday, 14:00 UTC). Keep cleanup=true for drills.
    const schedulerRole = new iam.Role(this, 'SchedulerRole', { assumedBy: new iam.ServicePrincipal('scheduler.amazonaws.com') });
    machine.grantStartExecution(schedulerRole);
    new scheduler.CfnSchedule(this, 'MonthlyDrill', {
      name: 'dr-monthly-drill',
      scheduleExpression: 'cron(0 14 ? * MON#1 *)',
      flexibleTimeWindow: { mode: 'OFF' },
      target: {
        arn: machine.stateMachineArn,
        roleArn: schedulerRole.roleArn,
        input: JSON.stringify({ resource_type: 'EC2', cleanup: true }),
      },
    });

    new cdk.CfnOutput(this, 'StateMachineArn', { value: machine.stateMachineArn });
    new cdk.CfnOutput(this, 'DrVaultName', { value: vault.backupVaultName });
  }
}

import * as cdk from 'aws-cdk-lib';
import { Construct } from 'constructs';
import * as backup from 'aws-cdk-lib/aws-backup';
import * as ec2 from 'aws-cdk-lib/aws-ec2';
import * as events from 'aws-cdk-lib/aws-events';
import * as iam from 'aws-cdk-lib/aws-iam';

export interface PrimaryRegionStackProps extends cdk.StackProps {
  drRegion: string;
  drVaultName: string;
  withDemoInstance: boolean;
}

/** Back up everything tagged dr=true daily and copy each recovery point to the DR region. */
export class PrimaryRegionStack extends cdk.Stack {
  constructor(scope: Construct, id: string, props: PrimaryRegionStackProps) {
    super(scope, id, props);

    const vault = new backup.BackupVault(this, 'PrimaryVault', {
      backupVaultName: 'primary-vault',
      removalPolicy: cdk.RemovalPolicy.DESTROY,
    });

    const drVault = backup.BackupVault.fromBackupVaultArn(
      this,
      'DrVault',
      `arn:${this.partition}:backup:${props.drRegion}:${this.account}:backup-vault:${props.drVaultName}`,
    );

    const plan = new backup.BackupPlan(this, 'PilotLightPlan', {
      backupPlanName: 'pilot-light-daily',
      backupVault: vault,
    });
    plan.addRule(
      new backup.BackupPlanRule({
        ruleName: 'daily-with-cross-region-copy',
        scheduleExpression: events.Schedule.cron({ hour: '5', minute: '0' }),
        startWindow: cdk.Duration.hours(1),
        completionWindow: cdk.Duration.hours(4),
        deleteAfter: cdk.Duration.days(7),
        copyActions: [{ destinationBackupVault: drVault, deleteAfter: cdk.Duration.days(14) }],
      }),
    );
    plan.addSelection('TaggedResources', {
      resources: [backup.BackupResource.fromTag('dr', 'true')],
      allowRestores: true,
    });

    // ---------------------------------------------------------------- demo workload to protect
    if (props.withDemoInstance) {
      const vpc = new ec2.Vpc(this, 'AppVpc', {
        maxAzs: 1,
        natGateways: 0,
        subnetConfiguration: [{ name: 'public', subnetType: ec2.SubnetType.PUBLIC }],
      });
      const role = new iam.Role(this, 'AppRole', {
        assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com'),
        managedPolicies: [iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore')],
      });
      const userData = ec2.UserData.forLinux();
      userData.addCommands(
        'dnf install -y nginx',
        'echo "<h1>Order service</h1><p>Restored from backup if you are reading this in the DR region.</p>" > /usr/share/nginx/html/index.html',
        'systemctl enable --now nginx',
      );
      const instance = new ec2.Instance(this, 'OrderService', {
        vpc,
        instanceType: ec2.InstanceType.of(ec2.InstanceClass.T4G, ec2.InstanceSize.NANO),
        machineImage: ec2.MachineImage.latestAmazonLinux2023({ cpuType: ec2.AmazonLinuxCpuType.ARM_64 }),
        role,
        userData,
        requireImdsv2: true,
        blockDevices: [{ deviceName: '/dev/xvda', volume: ec2.BlockDeviceVolume.ebs(8, { encrypted: true }) }],
      });
      cdk.Tags.of(instance).add('dr', 'true');
      new cdk.CfnOutput(this, 'ProtectedInstanceId', { value: instance.instanceId });
    }

    new cdk.CfnOutput(this, 'BackupPlanId', { value: plan.backupPlanId });
  }
}

"""DR restore drill steps, orchestrated by Step Functions.

Each call receives {"action": ..., "state": {...}} and returns the updated state.
Input to the state machine: {"resource_type": "EC2", "cleanup": true}
"""
import datetime as dt
import json
import os
import urllib.request

import boto3

backup = boto3.client("backup")
ec2 = boto3.client("ec2")
cw = boto3.client("cloudwatch")
sns = boto3.client("sns")

VAULT = os.environ["VAULT_NAME"]
SUBNET = os.environ["SUBNET_ID"]
SG = os.environ["SECURITY_GROUP_ID"]
ROLE = os.environ["RESTORE_ROLE_ARN"]
TOPIC = os.environ["TOPIC_ARN"]


def handler(event, _context):
    action, state = event["action"], event.get("state") or {}
    return globals()[action](state)


def now():
    return dt.datetime.now(dt.timezone.utc)


def find_latest(state):
    rtype = state.get("resource_type", "EC2")
    points = []
    for page in backup.get_paginator("list_recovery_points_by_backup_vault").paginate(
        BackupVaultName=VAULT, ByResourceType=rtype
    ):
        points += [p for p in page["RecoveryPoints"] if p["Status"] == "COMPLETED"]
    if not points:
        raise RuntimeError(f"No completed {rtype} recovery points in {VAULT} yet - wait for the first copy")
    latest = max(points, key=lambda p: p["CreationDate"])
    return {
        **state,
        "drill_started_at": now().isoformat(),
        "recovery_point_arn": latest["RecoveryPointArn"],
        "recovery_point_age_hours": round((now() - latest["CreationDate"]).total_seconds() / 3600, 1),
        "health_checks": 0,
    }


def start_restore(state):
    meta = backup.get_recovery_point_restore_metadata(
        BackupVaultName=VAULT, RecoveryPointArn=state["recovery_point_arn"]
    )["RestoreMetadata"]
    # Re-home the instance into the pilot-light network.
    for key in ("NetworkInterfaces", "CpuOptions", "SubnetId", "SecurityGroupIds", "Placement"):
        meta.pop(key, None)
    meta["SubnetId"] = SUBNET
    meta["SecurityGroupIds"] = json.dumps([SG])
    meta.pop("IamInstanceProfileName", None)  # instance profiles are regional-agnostic but may not exist; add if needed
    job = backup.start_restore_job(
        RecoveryPointArn=state["recovery_point_arn"], Metadata=meta, IamRoleArn=ROLE, ResourceType="EC2"
    )
    return {**state, "restore_job_id": job["RestoreJobId"], "restore_status": "PENDING"}


def check_restore(state):
    job = backup.describe_restore_job(RestoreJobId=state["restore_job_id"])
    out = {**state, "restore_status": job["Status"]}
    if job["Status"] == "COMPLETED":
        out["instance_id"] = job["CreatedResourceArn"].split("/")[-1]
        out["restore_completed_at"] = now().isoformat()
        ec2.create_tags(Resources=[out["instance_id"]], Tags=[{"Key": "dr-drill", "Value": "true"}])
    elif job["Status"] in ("FAILED", "ABORTED"):
        out["error"] = job.get("StatusMessage")
    return out


def validate(state):
    iid = state["instance_id"]
    statuses = ec2.describe_instance_status(InstanceIds=[iid], IncludeAllInstances=True)["InstanceStatuses"]
    ok = bool(statuses) and statuses[0]["InstanceState"]["Name"] == "running" and \
        statuses[0]["InstanceStatus"]["Status"] == "ok" and statuses[0]["SystemStatus"]["Status"] == "ok"
    http_ok = False
    if ok:
        res = ec2.describe_instances(InstanceIds=[iid])["Reservations"][0]["Instances"][0]
        ip = res.get("PublicIpAddress")
        if ip:
            try:
                http_ok = urllib.request.urlopen(f"http://{ip}/", timeout=5).status == 200
            except Exception:
                http_ok = False
        else:
            http_ok = True  # private-only restore: status checks are the signal
    healthy = ok and http_ok
    out = {**state, "healthy": healthy, "health_checks": state.get("health_checks", 0) + 1}
    if healthy:
        out["healthy_at"] = now().isoformat()
    return out


def cleanup_and_report(state):
    started = dt.datetime.fromisoformat(state["drill_started_at"])
    rto_min = round((now() - started).total_seconds() / 60, 1)
    rpo_h = state.get("recovery_point_age_hours")
    passed = bool(state.get("healthy"))

    cw.put_metric_data(Namespace="DisasterRecovery", MetricData=[
        {"MetricName": "DrillRtoMinutes", "Value": rto_min, "Unit": "None"},
        {"MetricName": "DrillRpoHours", "Value": rpo_h or 0, "Unit": "None"},
        {"MetricName": "DrillPassed", "Value": 1 if passed else 0, "Unit": "Count"},
    ])
    if state.get("cleanup", True) and state.get("instance_id"):
        ec2.terminate_instances(InstanceIds=[state["instance_id"]])

    report = {
        "result": "PASS" if passed else "FAIL (instance never became healthy)",
        "measured_rto_minutes": rto_min,
        "rpo_hours_at_drill_time": rpo_h,
        "recovery_point": state["recovery_point_arn"],
        "restored_instance": state.get("instance_id"),
        "cleaned_up": bool(state.get("cleanup", True)),
    }
    sns.publish(TopicArn=TOPIC, Subject=f"DR drill {report['result'][:4]}: RTO {rto_min} min"[:100],
                Message=json.dumps(report, indent=2))
    return {**state, "report": report}

{
  "app": "npx ts-node --prefer-ts-exts bin/app.ts",
  "watch": { "include": ["**"], "exclude": ["README.md", "cdk*.json", "**/*.d.ts", "**/*.js", "node_modules", "cdk.out"] },
  "context": {}
}

{
  "name": "14-low-cost-disaster-recovery",
  "version": "1.0.0",
  "private": true,
  "bin": { "app": "bin/app.js" },
  "scripts": {
    "build": "tsc",
    "synth": "cdk synth",
    "deploy": "cdk deploy --all",
    "destroy": "cdk destroy --all"
  },
  "devDependencies": {
    "@types/node": "^22.0.0",
    "aws-cdk": "^2.1000.0",
    "ts-node": "^10.9.2",
    "typescript": "~5.6.3"
  },
  "dependencies": {
    "aws-cdk-lib": "^2.200.0",
    "constructs": "^10.4.2"
  }
}

{
  "compilerOptions": {
    "target": "ES2022",
    "module": "commonjs",
    "lib": ["es2022"],
    "strict": true,
    "esModuleInterop": true,
    "skipLibCheck": true,
    "noEmit": true,
    "types": ["node"]
  },
  "exclude": ["node_modules", "cdk.out"]
}