# 03 · Chaos Engineering Game Day Platform

**Format:** AWS CDK (TypeScript) · **Est. cost:** ~$25/month while deployed (ALB + 2× t4g.micro). FIS actions bill per action-minute (~$0.10).

A production-shaped web tier plus a **library of AWS Fault Injection Service (FIS) experiments**, each with an automatic
stop condition, so you can run game days safely and measure recovery.

```
Internet ─▶ ALB ─▶ Auto Scaling group (2–4 instances, 2 AZs, ELB health checks)
                     ▲
     FIS experiments │  terminate 50% · CPU stress · +400ms latency · AZ network blackout
                     │
CloudWatch alarm (5xx rate > 20% for 2 min) ──▶ STOPS every experiment + SNS alert
Dashboard: error rate · healthy hosts · p95 latency · requests
```

## Deploy
```bash
npm install
npx cdk deploy -c alertEmail=you@example.com
```
Outputs include `AppUrl`, `DashboardUrl` and one `...TemplateId` per experiment.

## Run the game day
1. **Hypothesis** - write it down, e.g. *"Losing half the fleet causes < 1% errors and recovers in < 3 min."*
2. **Generate load** (separate terminal):
   ```bash
   URL=<AppUrl>; while true; do curl -s -o /dev/null -w "%{http_code}\n" $URL; sleep 0.2; done
   ```
3. **Inject**:
   ```bash
   aws fis start-experiment --experiment-template-id <TerminateHalfTemplateId>
   ```
4. **Observe** the `chaos-game-day` dashboard. Record time-to-detect, time-to-recover, error budget used.
5. **Learn** - fix what broke (health-check thresholds, deregistration delay, min capacity), then re-run.

| Experiment | What it proves |
|---|---|
| `TerminateHalf` | ASG + ELB health checks replace capacity without user-visible errors |
| `CpuStress` | Target-tracking scaling reacts fast enough under saturation |
| `NetworkLatency` | Timeouts, retries and health-check thresholds handle a slow node |
| `AzBlackout` | The app survives losing an entire Availability Zone |

## Results template
| Experiment | Hypothesis | Detect (s) | Recover (s) | Peak error % | Pass? | Fix |
|---|---|---|---|---|---|---|

## Clean up
```bash
npx cdk destroy
```

#!/usr/bin/env node
import * as cdk from 'aws-cdk-lib';
import { ChaosGameDayStack } from '../lib/chaos-game-day-stack';

const app = new cdk.App();
new ChaosGameDayStack(app, 'ChaosGameDay', {
  env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION },
  alertEmail: app.node.tryGetContext('alertEmail'),
});

import * as cdk from 'aws-cdk-lib';
import { Construct } from 'constructs';
import * as autoscaling from 'aws-cdk-lib/aws-autoscaling';
import * as cloudwatch from 'aws-cdk-lib/aws-cloudwatch';
import * as cwActions from 'aws-cdk-lib/aws-cloudwatch-actions';
import * as ec2 from 'aws-cdk-lib/aws-ec2';
import * as elbv2 from 'aws-cdk-lib/aws-elasticloadbalancingv2';
import * as fis from 'aws-cdk-lib/aws-fis';
import * as iam from 'aws-cdk-lib/aws-iam';
import * as logs from 'aws-cdk-lib/aws-logs';
import * as sns from 'aws-cdk-lib/aws-sns';
import * as subs from 'aws-cdk-lib/aws-sns-subscriptions';

export interface ChaosGameDayStackProps extends cdk.StackProps {
  alertEmail?: string;
}

/**
 * A small but production-shaped web tier (ALB + multi-AZ Auto Scaling group) plus a
 * library of AWS Fault Injection Service experiments with automatic stop conditions.
 */
export class ChaosGameDayStack extends cdk.Stack {
  constructor(scope: Construct, id: string, props: ChaosGameDayStackProps) {
    super(scope, id, props);

    // ---------------------------------------------------------------- target app
    // Public subnets only (no NAT gateway) keeps the lab cheap; instances only accept ALB traffic.
    const vpc = new ec2.Vpc(this, 'Vpc', {
      maxAzs: 2,
      natGateways: 0,
      subnetConfiguration: [{ name: 'public', subnetType: ec2.SubnetType.PUBLIC, cidrMask: 24 }],
    });

    const albSg = new ec2.SecurityGroup(this, 'AlbSg', { vpc, description: 'ALB ingress' });
    albSg.addIngressRule(ec2.Peer.anyIpv4(), ec2.Port.tcp(80), 'HTTP from the internet');
    const appSg = new ec2.SecurityGroup(this, 'AppSg', { vpc, description: 'App instances' });
    appSg.addIngressRule(albSg, ec2.Port.tcp(80), 'HTTP from ALB only');

    const role = new iam.Role(this, 'InstanceRole', {
      assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com'),
      managedPolicies: [iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore')],
    });

    const userData = ec2.UserData.forLinux();
    userData.addCommands(
      'dnf install -y nginx',
      'TOKEN=$(curl -s -X PUT http://169.254.169.254/latest/api/token -H "X-aws-ec2-metadata-token-ttl-seconds: 60")',
      'AZ=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/placement/availability-zone)',
      'IID=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-id)',
      'echo "<h1>Chaos game day</h1><p>Served by $IID in $AZ</p>" > /usr/share/nginx/html/index.html',
      'echo ok > /usr/share/nginx/html/health',
      'systemctl enable --now nginx',
    );

    const asg = new autoscaling.AutoScalingGroup(this, 'Asg', {
      vpc,
      vpcSubnets: { subnetType: ec2.SubnetType.PUBLIC },
      associatePublicIpAddress: true,
      instanceType: ec2.InstanceType.of(ec2.InstanceClass.T4G, ec2.InstanceSize.MICRO),
      machineImage: ec2.MachineImage.latestAmazonLinux2023({ cpuType: ec2.AmazonLinuxCpuType.ARM_64 }),
      securityGroup: appSg,
      role,
      userData,
      minCapacity: 2,
      maxCapacity: 4,
      healthChecks: autoscaling.HealthChecks.withAdditionalChecks({
        additionalTypes: [autoscaling.AdditionalHealthCheckType.ELB],
        gracePeriod: cdk.Duration.seconds(90),
      }),
      requireImdsv2: true,
    });
    cdk.Tags.of(asg).add('chaos-target', 'true', { applyToLaunchedInstances: true });
    asg.scaleOnCpuUtilization('CpuScaling', { targetUtilizationPercent: 60 });

    const alb = new elbv2.ApplicationLoadBalancer(this, 'Alb', { vpc, internetFacing: true, securityGroup: albSg });
    const listener = alb.addListener('Http', { port: 80, open: false });
    const tg = listener.addTargets('App', {
      port: 80,
      targets: [asg],
      healthCheck: { path: '/health', interval: cdk.Duration.seconds(10), healthyThresholdCount: 2, unhealthyThresholdCount: 2 },
      deregistrationDelay: cdk.Duration.seconds(10),
    });

    // ---------------------------------------------------------------- guard rails
    const topic = new sns.Topic(this, 'GameDayTopic', { displayName: 'Chaos game day' });
    if (props.alertEmail) topic.addSubscription(new subs.EmailSubscription(props.alertEmail));

    const errorRate = new cloudwatch.MathExpression({
      label: '5xx error rate %',
      expression: '100 * (e5 + t5) / MAX([req, 1])',
      usingMetrics: {
        e5: alb.metrics.httpCodeElb(elbv2.HttpCodeElb.ELB_5XX_COUNT, { statistic: 'Sum' }),
        t5: tg.metrics.httpCodeTarget(elbv2.HttpCodeTarget.TARGET_5XX_COUNT, { statistic: 'Sum' }),
        req: alb.metrics.requestCount({ statistic: 'Sum' }),
      },
      period: cdk.Duration.minutes(1),
    });

    // Stop condition: if customers feel real pain, FIS halts every experiment automatically.
    const stopAlarm = new cloudwatch.Alarm(this, 'StopConditionAlarm', {
      alarmName: 'chaos-stop-condition-5xx',
      alarmDescription: 'Error rate above 20% for 2 minutes - abort chaos experiments',
      metric: errorRate,
      threshold: 20,
      evaluationPeriods: 2,
      treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
    });
    stopAlarm.addAlarmAction(new cwActions.SnsAction(topic));

    const unhealthy = new cloudwatch.Alarm(this, 'UnhealthyHostsAlarm', {
      alarmName: 'chaos-unhealthy-hosts',
      metric: tg.metrics.unhealthyHostCount({ period: cdk.Duration.minutes(1) }),
      threshold: 1,
      evaluationPeriods: 1,
      comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
    });
    unhealthy.addAlarmAction(new cwActions.SnsAction(topic));

    // ---------------------------------------------------------------- FIS
    const fisLogs = new logs.LogGroup(this, 'FisLogs', {
      logGroupName: '/chaos/fis-experiments',
      retention: logs.RetentionDays.THREE_MONTHS,
      removalPolicy: cdk.RemovalPolicy.DESTROY,
    });

    const fisRole = new iam.Role(this, 'FisRole', {
      assumedBy: new iam.ServicePrincipal('fis.amazonaws.com'),
      managedPolicies: [
        iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSFaultInjectionSimulatorEC2Access'),
        iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSFaultInjectionSimulatorSSMAccess'),
        iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSFaultInjectionSimulatorNetworkAccess'),
      ],
    });
    fisRole.addToPolicy(
      new iam.PolicyStatement({
        actions: ['logs:CreateLogDelivery', 'logs:PutResourcePolicy', 'logs:DescribeResourcePolicies', 'logs:DescribeLogGroups'],
        resources: ['*'],
      }),
    );
    fisLogs.grantWrite(fisRole);

    const common = {
      roleArn: fisRole.roleArn,
      stopConditions: [{ source: 'aws:cloudwatch:alarm', value: stopAlarm.alarmArn }],
      logConfiguration: { logSchemaVersion: 2, cloudWatchLogsConfiguration: { LogGroupArn: fisLogs.logGroupArn } },
    };
    const taggedInstances = (mode: string) => ({
      resourceType: 'aws:ec2:instance',
      resourceTags: { 'chaos-target': 'true' },
      filters: [{ path: 'State.Name', values: ['running'] }],
      selectionMode: mode,
    });
    const ssmDoc = (name: string) => `arn:${this.partition}:ssm:${this.region}::document/${name}`;

    const experiments: Record<string, fis.CfnExperimentTemplateProps> = {
      TerminateHalf: {
        ...common,
        description: 'Terminate 50% of web instances - does the ASG self-heal with no 5xx spike?',
        targets: { Web: taggedInstances('PERCENT(50)') },
        actions: { Terminate: { actionId: 'aws:ec2:terminate-instances', targets: { Instances: 'Web' } } },
        tags: { Name: 'gameday-terminate-half' },
      },
      CpuStress: {
        ...common,
        description: 'Burn 100% CPU on all web instances for 5 minutes - does scaling kick in?',
        targets: { Web: taggedInstances('ALL') },
        actions: {
          Stress: {
            actionId: 'aws:ssm:send-command',
            parameters: {
              documentArn: ssmDoc('AWSFIS-Run-CPU-Stress'),
              documentParameters: JSON.stringify({ DurationSeconds: '300', InstallDependencies: 'True' }),
              duration: 'PT6M',
            },
            targets: { Instances: 'Web' },
          },
        },
        tags: { Name: 'gameday-cpu-stress' },
      },
      NetworkLatency: {
        ...common,
        description: 'Add 400ms latency on one instance - do health checks and timeouts behave?',
        targets: { Web: taggedInstances('COUNT(1)') },
        actions: {
          Latency: {
            actionId: 'aws:ssm:send-command',
            parameters: {
              documentArn: ssmDoc('AWSFIS-Run-Network-Latency'),
              documentParameters: JSON.stringify({ DelayMilliseconds: '400', DurationSeconds: '300', InstallDependencies: 'True' }),
              duration: 'PT6M',
            },
            targets: { Instances: 'Web' },
          },
        },
        tags: { Name: 'gameday-network-latency' },
      },
      AzBlackout: {
        ...common,
        description: 'Cut all network traffic in one AZ subnet for 5 minutes - simulated AZ outage',
        targets: {
          OneAz: {
            resourceType: 'aws:ec2:subnet',
            resourceArns: [
              `arn:${this.partition}:ec2:${this.region}:${this.account}:subnet/${vpc.publicSubnets[0].subnetId}`,
            ],
            selectionMode: 'ALL',
          },
        },
        actions: {
          Disrupt: {
            actionId: 'aws:network:disrupt-connectivity',
            parameters: { duration: 'PT5M', scope: 'all' },
            targets: { Subnets: 'OneAz' },
          },
        },
        tags: { Name: 'gameday-az-blackout' },
      },
    };

    for (const [name, props] of Object.entries(experiments)) {
      const tpl = new fis.CfnExperimentTemplate(this, name, props);
      new cdk.CfnOutput(this, `${name}TemplateId`, { value: tpl.attrId });
    }

    // ---------------------------------------------------------------- observe
    const dash = new cloudwatch.Dashboard(this, 'Dashboard', { dashboardName: 'chaos-game-day' });
    dash.addWidgets(
      new cloudwatch.GraphWidget({ title: 'Error rate %', left: [errorRate], leftAnnotations: [{ value: 20, label: 'stop' }] }),
      new cloudwatch.GraphWidget({
        title: 'Healthy vs unhealthy hosts',
        left: [tg.metrics.healthyHostCount(), tg.metrics.unhealthyHostCount()],
      }),
      new cloudwatch.GraphWidget({ title: 'p95 latency', left: [tg.metrics.targetResponseTime({ statistic: 'p95' })] }),
      new cloudwatch.GraphWidget({ title: 'Requests', left: [alb.metrics.requestCount()] }),
    );

    new cdk.CfnOutput(this, 'AppUrl', { value: `http://${alb.loadBalancerDnsName}` });
    new cdk.CfnOutput(this, 'DashboardUrl', {
      value: `https://${this.region}.console.aws.amazon.com/cloudwatch/home?region=${this.region}#dashboards:name=chaos-game-day`,
    });
  }
}

{
  "app": "npx ts-node --prefer-ts-exts bin/app.ts",
  "watch": { "include": ["**"], "exclude": ["README.md", "cdk*.json", "**/*.d.ts", "**/*.js", "node_modules", "cdk.out"] },
  "context": {}
}

{
  "name": "03-chaos-engineering-game-day",
  "version": "1.0.0",
  "private": true,
  "bin": { "app": "bin/app.js" },
  "scripts": {
    "build": "tsc",
    "synth": "cdk synth",
    "deploy": "cdk deploy --all",
    "destroy": "cdk destroy --all"
  },
  "devDependencies": {
    "@types/node": "^22.0.0",
    "aws-cdk": "^2.1000.0",
    "ts-node": "^10.9.2",
    "typescript": "~5.6.3"
  },
  "dependencies": {
    "aws-cdk-lib": "^2.200.0",
    "constructs": "^10.4.2"
  }
}

{
  "compilerOptions": {
    "target": "ES2022",
    "module": "commonjs",
    "lib": ["es2022"],
    "strict": true,
    "esModuleInterop": true,
    "skipLibCheck": true,
    "noEmit": true,
    "types": ["node"]
  },
  "exclude": ["node_modules", "cdk.out"]
}