# 12 · Multi-Cloud Observability Hub

**Format:** AWS CDK (TypeScript) + OpenTelemetry collector config · **Est. cost:** Managed Grafana $9/editor/month; AMP ~$0.90 per 10M samples ingested (small fleets: a few $/month)

**One pane of glass for AWS, Azure and GCP**, built on open standards so no vendor lock-in:

```
 AWS (EC2/ECS/EKS)          Azure VMs                     GCP VMs
 ADOT collector             otelcol + Entra ID token      otelcol + Google ID token
     │ IAM role                 │ OIDC → STS                  │ web identity → STS     (no stored AWS keys)
     └──────────────┬──────────┴─────────────┬──────────────┘
                    ▼                        ▼
      Amazon Managed Prometheus      X-Ray / CloudWatch Logs
      (recording + alert rules)              │
                    └──────────┬─────────────┘
                               ▼
                 Amazon Managed Grafana (SSO)  ◀── CloudWatch cross-account (OAM sink)
```
Every series carries a `cloud_provider` label, so one dashboard compares all three clouds side by side.

## Deploy
```bash
npm install
npx cdk deploy \
  -c gcpServiceAccountUniqueId=123456789012345678901 \
  -c azureTenantId=<tenant-guid> -c azureAppId=<app-registration-client-id> -c azurePrincipalId=<managed-identity-object-id> \
  -c sourceAccountIds=111111111111,222222222222
```
- Grafana needs **IAM Identity Center**. Use `-c enableGrafana=false` to skip it.
- Each external cloud is optional; omit its context values to skip it.

## Install the collector (any cloud)
```bash
# GCP / Azure VM
sudo cp collector/otel-collector-config.yaml /etc/otel/config.yaml
sudo cp collector/credential-process-gcp.sh /etc/otel/      # or -azure.sh
export CLOUD_PROVIDER=gcp AWS_REGION=us-east-1 AWS_CONFIG_FILE=/etc/otel/aws-config \
       REMOTE_WRITE_URL=<RemoteWriteUrl>
otelcol-contrib --config /etc/otel/config.yaml
```
On AWS, attach the `observability-hub-collector` managed policy to the instance/task role instead.

## Prove it
In Grafana → Explore → Prometheus:
```promql
cloud:cpu_utilization:avg
count by (cloud_provider) (up)
```
You'll see one line per cloud. Stop a VM on any cloud and the `HostDown` alert fires within 5 minutes.

## Clean up
```bash
npx cdk destroy
```

#!/usr/bin/env node
import * as cdk from 'aws-cdk-lib';
import { ObservabilityHubStack } from '../lib/observability-hub-stack';

const app = new cdk.App();
const ctx = (k: string) => app.node.tryGetContext(k) as string | undefined;

new ObservabilityHubStack(app, 'ObservabilityHub', {
  env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION },
  enableGrafana: ctx('enableGrafana') !== 'false', // needs IAM Identity Center
  gcpServiceAccountUniqueId: ctx('gcpServiceAccountUniqueId'),
  azure: ctx('azureTenantId') && ctx('azureAppId') && ctx('azurePrincipalId')
    ? { tenantId: ctx('azureTenantId')!, appId: ctx('azureAppId')!, principalId: ctx('azurePrincipalId')! }
    : undefined,
  sourceAccountIds: ctx('sourceAccountIds')?.split(',').filter(Boolean) ?? [],
});

import * as cdk from 'aws-cdk-lib';
import { Construct } from 'constructs';
import * as aps from 'aws-cdk-lib/aws-aps';
import * as grafana from 'aws-cdk-lib/aws-grafana';
import * as iam from 'aws-cdk-lib/aws-iam';
import * as logs from 'aws-cdk-lib/aws-logs';
import * as oam from 'aws-cdk-lib/aws-oam';

export interface ObservabilityHubStackProps extends cdk.StackProps {
  enableGrafana: boolean;
  /** GCP service account "uniqueId" (numeric) whose identity tokens may assume the writer role. */
  gcpServiceAccountUniqueId?: string;
  /** Azure Entra ID app registration + managed identity / service principal object ID. */
  azure?: { tenantId: string; appId: string; principalId: string };
  /** AWS accounts that share CloudWatch telemetry into this monitoring account. */
  sourceAccountIds: string[];
}

/**
 * One pane of glass for AWS + Azure + GCP, built on open standards:
 * OpenTelemetry collectors everywhere -> Amazon Managed Prometheus -> Amazon Managed Grafana.
 * External clouds authenticate with keyless OIDC federation (no AWS access keys on other clouds).
 */
export class ObservabilityHubStack extends cdk.Stack {
  constructor(scope: Construct, id: string, props: ObservabilityHubStackProps) {
    super(scope, id, props);

    const amp = new aps.CfnWorkspace(this, 'Prometheus', { alias: 'multi-cloud-hub' });

    new aps.CfnRuleGroupsNamespace(this, 'Rules', {
      workspace: amp.attrArn,
      name: 'multi-cloud',
      data: [
        'groups:',
        '  - name: fleet',
        '    rules:',
        '      - record: cloud:cpu_utilization:avg',
        '        expr: avg by (cloud_provider) (1 - rate(system_cpu_time_seconds_total{state="idle"}[5m]))',
        '      - alert: HostDown',
        '        expr: up == 0',
        '        for: 5m',
        '        labels: { severity: critical }',
        '        annotations: { summary: "{{ $labels.instance }} ({{ $labels.cloud_provider }}) is down" }',
        '      - alert: HighCpu',
        '        expr: (1 - rate(system_cpu_time_seconds_total{state="idle"}[5m])) > 0.9',
        '        for: 10m',
        '        labels: { severity: warning }',
      ].join('\n'),
    });

    const otlpLogs = new logs.LogGroup(this, 'ExternalLogs', {
      logGroupName: '/observability-hub/external-clouds',
      retention: logs.RetentionDays.ONE_MONTH,
      removalPolicy: cdk.RemovalPolicy.DESTROY,
    });

    const writerPermissions = new iam.PolicyDocument({
      statements: [
        new iam.PolicyStatement({ actions: ['aps:RemoteWrite'], resources: [amp.attrArn] }),
        new iam.PolicyStatement({ actions: ['xray:PutTraceSegments', 'xray:PutTelemetryRecords'], resources: ['*'] }),
        new iam.PolicyStatement({
          actions: ['logs:CreateLogStream', 'logs:PutLogEvents', 'logs:DescribeLogStreams'],
          resources: [otlpLogs.logGroupArn, `${otlpLogs.logGroupArn}:*`],
        }),
      ],
    });

    // ---------------------------------------------------------------- AWS-side collectors (EC2/ECS/EKS)
    new iam.ManagedPolicy(this, 'AwsCollectorPolicy', {
      managedPolicyName: 'observability-hub-collector',
      document: writerPermissions,
      description: 'Attach to ADOT collector roles running inside AWS',
    });

    // ---------------------------------------------------------------- GCP: keyless via Google identity tokens
    if (props.gcpServiceAccountUniqueId) {
      const gcpRole = new iam.Role(this, 'GcpCollectorRole', {
        roleName: 'observability-hub-gcp-collector',
        maxSessionDuration: cdk.Duration.hours(1),
        assumedBy: new iam.WebIdentityPrincipal('accounts.google.com', {
          StringEquals: {
            'accounts.google.com:aud': props.gcpServiceAccountUniqueId, // token "azp" = the service account
            'accounts.google.com:oaud': 'sts.amazonaws.com', // token "aud" requested by the collector
          },
        }),
        inlinePolicies: { write: writerPermissions },
      });
      new cdk.CfnOutput(this, 'GcpCollectorRoleArn', { value: gcpRole.roleArn });
    }

    // ---------------------------------------------------------------- Azure: keyless via Entra ID OIDC
    if (props.azure) {
      const issuer = `login.microsoftonline.com/${props.azure.tenantId}/v2.0`;
      const provider = new iam.OpenIdConnectProvider(this, 'AzureOidc', {
        url: `https://${issuer}`,
        clientIds: [props.azure.appId],
      });
      const azureRole = new iam.Role(this, 'AzureCollectorRole', {
        roleName: 'observability-hub-azure-collector',
        maxSessionDuration: cdk.Duration.hours(1),
        assumedBy: new iam.OpenIdConnectPrincipal(provider, {
          StringEquals: { [`${issuer}:aud`]: props.azure.appId, [`${issuer}:sub`]: props.azure.principalId },
        }),
        inlinePolicies: { write: writerPermissions },
      });
      new cdk.CfnOutput(this, 'AzureCollectorRoleArn', { value: azureRole.roleArn });
    }

    // ---------------------------------------------------------------- AWS multi-account CloudWatch
    if (props.sourceAccountIds.length) {
      const sink = new oam.CfnSink(this, 'MonitoringSink', {
        name: 'observability-hub',
        policy: {
          Version: '2012-10-17',
          Statement: [
            {
              Effect: 'Allow',
              Principal: { AWS: props.sourceAccountIds },
              Action: ['oam:CreateLink', 'oam:UpdateLink'],
              Resource: '*',
              Condition: {
                'ForAllValues:StringEquals': {
                  'oam:ResourceTypes': ['AWS::CloudWatch::Metric', 'AWS::Logs::LogGroup', 'AWS::XRay::Trace'],
                },
              },
            },
          ],
        },
      });
      new cdk.CfnOutput(this, 'OamSinkArn', { value: sink.attrArn });
    }

    // ---------------------------------------------------------------- Grafana
    if (props.enableGrafana) {
      const grafanaRole = new iam.Role(this, 'GrafanaRole', {
        assumedBy: new iam.ServicePrincipal('grafana.amazonaws.com', {
          conditions: { StringEquals: { 'aws:SourceAccount': this.account } },
        }),
        managedPolicies: [
          iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AmazonGrafanaCloudWatchAccess'),
          iam.ManagedPolicy.fromAwsManagedPolicyName('AWSXrayReadOnlyAccess'),
        ],
      });
      grafanaRole.addToPolicy(
        new iam.PolicyStatement({
          actions: ['aps:ListWorkspaces', 'aps:DescribeWorkspace', 'aps:QueryMetrics', 'aps:GetLabels', 'aps:GetSeries', 'aps:GetMetricMetadata'],
          resources: ['*'],
        }),
      );
      const ws = new grafana.CfnWorkspace(this, 'Grafana', {
        name: 'multi-cloud-hub',
        accountAccessType: 'CURRENT_ACCOUNT',
        authenticationProviders: ['AWS_SSO'],
        permissionType: 'CUSTOMER_MANAGED',
        roleArn: grafanaRole.roleArn,
        dataSources: ['PROMETHEUS', 'CLOUDWATCH', 'XRAY'],
        grafanaVersion: '10.4',
        pluginAdminEnabled: true,
      });
      new cdk.CfnOutput(this, 'GrafanaUrl', { value: `https://${ws.attrEndpoint}` });
    }

    new cdk.CfnOutput(this, 'RemoteWriteUrl', { value: `${amp.attrPrometheusEndpoint}api/v1/remote_write` });
    new cdk.CfnOutput(this, 'PrometheusQueryUrl', { value: `${amp.attrPrometheusEndpoint}api/v1/query` });
    new cdk.CfnOutput(this, 'ExternalLogGroup', { value: otlpLogs.logGroupName });
  }
}

#!/usr/bin/env bash
# AWS credential_process for an Azure VM with a managed identity: get an Entra ID token
# for the app registration and exchange it for short-lived AWS credentials.
#
# /etc/otel/aws-config:
#   [default]
#   credential_process = /etc/otel/credential-process-azure.sh <AZURE_APP_ID> arn:aws:iam::123456789012:role/observability-hub-azure-collector
set -euo pipefail
APP_ID="$1"
ROLE_ARN="$2"
TOKEN=$(curl -sf -H Metadata:true \
  "http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=api://${APP_ID}" \
  | python3 -c 'import json,sys;print(json.load(sys.stdin)["access_token"])')

aws sts assume-role-with-web-identity \
  --role-arn "$ROLE_ARN" \
  --role-session-name "azure-$(hostname)" \
  --web-identity-token "$TOKEN" \
  --duration-seconds 3600 \
  --query '{Version:`1`,AccessKeyId:Credentials.AccessKeyId,SecretAccessKey:Credentials.SecretAccessKey,SessionToken:Credentials.SessionToken,Expiration:Credentials.Expiration}' \
  --output json

#!/usr/bin/env bash
# AWS credential_process for a GCP VM: exchange the VM service account's Google identity
# token for short-lived AWS credentials. No AWS keys are ever stored on the VM.
#
# /etc/otel/aws-config:
#   [default]
#   credential_process = /etc/otel/credential-process-gcp.sh arn:aws:iam::123456789012:role/observability-hub-gcp-collector
#
# Trust (see stack): accounts.google.com:aud  == service account uniqueId (token "azp")
#                    accounts.google.com:oaud == sts.amazonaws.com        (token "aud")
set -euo pipefail
ROLE_ARN="$1"
TOKEN=$(curl -sf -H "Metadata-Flavor: Google" \
  "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity?audience=sts.amazonaws.com&format=full")

aws sts assume-role-with-web-identity \
  --role-arn "$ROLE_ARN" \
  --role-session-name "gcp-$(hostname)" \
  --web-identity-token "$TOKEN" \
  --duration-seconds 3600 \
  --query '{Version:`1`,AccessKeyId:Credentials.AccessKeyId,SecretAccessKey:Credentials.SecretAccessKey,SessionToken:Credentials.SessionToken,Expiration:Credentials.Expiration}' \
  --output json

# AWS Distro for OpenTelemetry (ADOT) / otelcol-contrib config - identical on AWS, Azure and GCP.
# Only CLOUD_PROVIDER, AWS_REGION and REMOTE_WRITE_URL differ per environment.
#   export CLOUD_PROVIDER=gcp   # aws | azure | gcp
#   export AWS_REGION=us-east-1
#   export REMOTE_WRITE_URL=https://aps-workspaces.us-east-1.amazonaws.com/workspaces/ws-xxxx/api/v1/remote_write
#   export AWS_CONFIG_FILE=/etc/otel/aws-config   # see credential-process scripts for Azure/GCP

extensions:
  sigv4auth:
    region: ${env:AWS_REGION}
    service: aps
  health_check: {}

receivers:
  hostmetrics:
    collection_interval: 30s
    scrapers: { cpu: {}, memory: {}, disk: {}, filesystem: {}, network: {}, load: {} }
  otlp:
    protocols:
      grpc: { endpoint: 0.0.0.0:4317 }
      http: { endpoint: 0.0.0.0:4318 }
  prometheus:
    config:
      scrape_configs:
        - job_name: local-apps
          scrape_interval: 30s
          static_configs:
            - targets: ['localhost:9100']   # node_exporter or your app's /metrics

processors:
  memory_limiter: { check_interval: 5s, limit_percentage: 80, spike_limit_percentage: 25 }
  resourcedetection:
    detectors: [env, system, ec2, gcp, azure]
    timeout: 5s
    override: false
  resource:
    attributes:
      - { key: cloud_provider, value: '${env:CLOUD_PROVIDER}', action: upsert }
  batch: { timeout: 10s, send_batch_size: 1024 }

exporters:
  prometheusremotewrite:
    endpoint: ${env:REMOTE_WRITE_URL}
    auth: { authenticator: sigv4auth }
    resource_to_telemetry_conversion: { enabled: true }
  awsxray:
    region: ${env:AWS_REGION}
  awscloudwatchlogs:
    region: ${env:AWS_REGION}
    log_group_name: /observability-hub/external-clouds
    log_stream_name: ${env:CLOUD_PROVIDER}-${env:HOSTNAME}

service:
  extensions: [sigv4auth, health_check]
  pipelines:
    metrics:
      receivers: [hostmetrics, otlp, prometheus]
      processors: [memory_limiter, resourcedetection, resource, batch]
      exporters: [prometheusremotewrite]
    traces:
      receivers: [otlp]
      processors: [memory_limiter, resourcedetection, resource, batch]
      exporters: [awsxray]
    logs:
      receivers: [otlp]
      processors: [memory_limiter, resourcedetection, resource, batch]
      exporters: [awscloudwatchlogs]

{
  "app": "npx ts-node --prefer-ts-exts bin/app.ts",
  "watch": { "include": ["**"], "exclude": ["README.md", "cdk*.json", "**/*.d.ts", "**/*.js", "node_modules", "cdk.out"] },
  "context": {}
}

{
  "name": "12-multi-cloud-observability-hub",
  "version": "1.0.0",
  "private": true,
  "bin": { "app": "bin/app.js" },
  "scripts": {
    "build": "tsc",
    "synth": "cdk synth",
    "deploy": "cdk deploy --all",
    "destroy": "cdk destroy --all"
  },
  "devDependencies": {
    "@types/node": "^22.0.0",
    "aws-cdk": "^2.1000.0",
    "ts-node": "^10.9.2",
    "typescript": "~5.6.3"
  },
  "dependencies": {
    "aws-cdk-lib": "^2.200.0",
    "constructs": "^10.4.2"
  }
}

{
  "compilerOptions": {
    "target": "ES2022",
    "module": "commonjs",
    "lib": ["es2022"],
    "strict": true,
    "esModuleInterop": true,
    "skipLibCheck": true,
    "noEmit": true,
    "types": ["node"]
  },
  "exclude": ["node_modules", "cdk.out"]
}