AWS Config Remediation with SSM Automation

Automatic vs manual remediation — when to use each

Automatic remediation fires the SSM document immediately when a resource transitions to NON_COMPLIANT — no human approval step. Manual remediation creates a remediation execution record that appears in the Config console and can be triggered by an operator clicking "Remediate" or by calling start-remediation-execution programmatically. The choice affects risk profile and auditability differently.

Configuring a remediation action with CDK

Remediation configurations reference an SSM document (either AWS-provided or custom) and map parameters from the Config finding (resource ID, resource type, account ID) to the document's input parameters. The RESOURCE_ID binding passes the ARN or ID of the non-compliant resource. AWS_ACCOUNT_ID and AWS_REGION bindings inject context values. Static values use the StaticValue type.

import * as cdk from "aws-cdk-lib";
import * as config from "aws-cdk-lib/aws-config";
import * as iam from "aws-cdk-lib/aws-iam";

// IAM role that SSM Automation executes as — must have write permissions for the fix
const remediationRole = new iam.Role(this, "ConfigRemediationRole", {
  assumedBy: new iam.ServicePrincipal("ssm.amazonaws.com"),
  description: "Role assumed by SSM Automation when Config triggers remediation",
});

// Grant permission to create VPC flow logs (for the flow logs remediation)
remediationRole.addToPolicy(new iam.PolicyStatement({
  actions: [
    "ec2:CreateFlowLogs",
    "ec2:DescribeFlowLogs",
    "iam:PassRole",             // needed to pass the flow logs delivery role to ec2:CreateFlowLogs
  ],
  resources: ["*"],
}));

// Grant permission to add tags (for the required-tags remediation)
remediationRole.addToPolicy(new iam.PolicyStatement({
  actions: [
    "dynamodb:TagResource",
    "lambda:TagResource",
    "ecs:TagResource",
    "ec2:CreateTags",
  ],
  resources: ["*"],
}));

// Remediation action for VPC flow logs — automatic (low risk)
const flowLogsDeliveryRoleArn = "arn:aws:iam::123456789012:role/VpcFlowLogsDelivery";

new config.CfnRemediationConfiguration(this, "VpcFlowLogsRemediation", {
  configRuleName: "mcp-vpc-flow-logs-enabled", // must match the rule name exactly
  targetType: "SSM_DOCUMENT",
  targetId: "AWSConfigRemediation-CreateVPCFlowLogToS3Bucket",
  automatic: true,
  maximumAutomaticAttempts: 3,     // stop after 3 failed attempts to prevent loops
  retryAttemptSeconds: 60,         // wait 60 seconds between retry attempts
  executionControls: {
    ssmControls: {
      concurrentExecutionRatePercentage: 25,  // at most 25% of non-compliant resources simultaneously
      errorPercentage: 20,                    // halt if >20% of executions fail
    },
  },
  parameters: {
    // VpcId: the non-compliant resource ID (the VPC ARN or ID)
    VpcId: { resourceValue: { value: "RESOURCE_ID" } },
    // S3BucketArn: static value — destination for flow log delivery
    S3BucketArn: {
      staticValue: { values: ["arn:aws:s3:::mcp-flow-logs-123456789012"] },
    },
    // AutomationAssumeRole: the IAM role SSM Automation uses to make API calls
    AutomationAssumeRole: {
      staticValue: { values: [remediationRole.roleArn] },
    },
  },
});

// Remediation action for required tags — automatic (tag addition is safe)
new config.CfnRemediationConfiguration(this, "RequiredTagsRemediation", {
  configRuleName: "mcp-required-tags",
  targetType: "SSM_DOCUMENT",
  targetId: "AWS-AddTagsToResource", // AWS-provided document for adding tags
  automatic: true,
  maximumAutomaticAttempts: 2,
  retryAttemptSeconds: 30,
  executionControls: {
    ssmControls: {
      concurrentExecutionRatePercentage: 50,
      errorPercentage: 10,
    },
  },
  parameters: {
    ResourceARN: { resourceValue: { value: "RESOURCE_ID" } },
    Tags: {
      staticValue: {
        values: [
          // Placeholder tags — signals the resource needs manual tag review
          '{"Environment":"requires-tagging","Service":"unknown","Owner":"ops@company.com"}',
        ],
      },
    },
    AutomationAssumeRole: {
      staticValue: { values: [remediationRole.roleArn] },
    },
  },
});

Custom SSM Automation document for MCP-specific remediation

When AWS-provided documents don't match your use case, write a custom SSM Automation document. For MCP servers, a common need is rotating the API key stored in a Lambda function's environment variable when the corresponding Secrets Manager secret has been updated — a fix that requires calling Lambda's UpdateFunctionConfiguration to inject the new secret value (or, better, removing the hardcoded value entirely and switching to Secrets Manager references).

# Custom SSM Automation document: remove hardcoded API keys from Lambda environment variables
# Upload to SSM with: aws ssm create-document --name McpRemediate-LambdaRemoveRawSecrets
# --document-type Automation --content file://document.yaml

schemaVersion: "0.3"
description: "Remove potentially-hardcoded secrets from Lambda function environment variables"
assumeRole: "{{ AutomationAssumeRole }}"

parameters:
  FunctionName:
    type: String
    description: "Lambda function name or ARN to remediate"
  AutomationAssumeRole:
    type: String
    description: "IAM role ARN for this automation to assume"
  SecretVariableNames:
    type: StringList
    description: "Environment variable names to remove from the function config"
    default:
      - "API_KEY"
      - "SECRET_KEY"
      - "DATABASE_PASSWORD"
      - "OPENAI_API_KEY"

mainSteps:

  - name: GetCurrentFunctionConfig
    action: aws:executeAwsApi
    inputs:
      Service: lambda
      Api: GetFunctionConfiguration
      FunctionName: "{{ FunctionName }}"
    outputs:
      - Name: CurrentEnvironmentVariables
        Selector: "$.Environment.Variables"
        Type: StringMap

  - name: BuildSanitizedEnvironment
    action: aws:executeScript
    inputs:
      Runtime: python3.11
      Handler: script_handler
      InputPayload:
        current_env: "{{ GetCurrentFunctionConfig.CurrentEnvironmentVariables }}"
        secret_keys: "{{ SecretVariableNames }}"
      Script: |
        def script_handler(events, context):
            current = events.get("current_env", {})
            keys_to_remove = events.get("secret_keys", [])
            sanitized = {k: v for k, v in current.items() if k not in keys_to_remove}
            removed = [k for k in keys_to_remove if k in current]
            return {
                "sanitized_env": sanitized,
                "removed_keys": removed,
                "removal_count": len(removed)
            }
    outputs:
      - Name: SanitizedEnvironment
        Selector: "$.Payload.sanitized_env"
        Type: StringMap
      - Name: RemovedKeys
        Selector: "$.Payload.removed_keys"
        Type: StringList
      - Name: RemovalCount
        Selector: "$.Payload.removal_count"
        Type: Integer

  - name: UpdateFunctionEnvironment
    action: aws:executeAwsApi
    inputs:
      Service: lambda
      Api: UpdateFunctionConfiguration
      FunctionName: "{{ FunctionName }}"
      Environment:
        Variables: "{{ BuildSanitizedEnvironment.SanitizedEnvironment }}"

  - name: WaitForUpdate
    action: aws:waitForAwsResourceProperty
    inputs:
      Service: lambda
      Api: GetFunctionConfiguration
      FunctionName: "{{ FunctionName }}"
      PropertySelector: "$.LastUpdateStatus"
      DesiredValues:
        - "Successful"
      MaxAttempts: 10
      DelaySeconds: 5

  - name: LogRemediation
    action: aws:executeAwsApi
    inputs:
      Service: cloudwatch
      Api: PutMetricData
      Namespace: MCP/ConfigRemediation
      MetricData:
        - MetricName: LambdaSecretsRemoved
          Value: "{{ BuildSanitizedEnvironment.RemovalCount }}"
          Unit: Count
          Dimensions:
            - Name: FunctionName
              Value: "{{ FunctionName }}"

Monitoring remediation execution results

Config stores the execution history of every remediation attempt. Query it with describe-remediation-execution-status to check whether remediations succeeded, failed, or are in progress. Failed executions include the SSM Automation execution ID, which you can look up in Systems Manager to get the full step-by-step failure output.

import {
  ConfigServiceClient,
  DescribeRemediationExecutionStatusCommand,
} from "@aws-sdk/client-config-service";
import {
  SSMClient,
  GetAutomationExecutionCommand,
} from "@aws-sdk/client-ssm";

const configClient = new ConfigServiceClient({ region: "us-east-1" });
const ssmClient = new SSMClient({ region: "us-east-1" });

async function checkRemediationStatus(ruleName: string): Promise<void> {
  let nextToken: string | undefined;
  do {
    const status = await configClient.send(
      new DescribeRemediationExecutionStatusCommand({
        ConfigRuleName: ruleName,
        NextToken: nextToken,
      })
    );

    for (const exec of status.RemediationExecutionStatuses ?? []) {
      const resourceKey = exec.ResourceKey;
      console.log(
        `Resource: ${resourceKey?.ResourceType}/${resourceKey?.ResourceId}`,
        `State: ${exec.State}`,
        `Last updated: ${exec.LastUpdatedTime?.toISOString()}`
      );

      if (exec.State === "FAILED" && exec.StepDetails) {
        for (const step of exec.StepDetails) {
          if (step.State === "FAILED") {
            console.error(`  Failed step: ${step.Name} — ${step.ErrorMessage}`);
            // Get SSM execution details for the specific failure
            if (step.StartTime) {
              // Note: Config doesn't directly expose the SSM execution ID in the SDK
              // Look up in the SSM console under Automation → Executions, filtered by time
              console.log(`  Check SSM Automation executions around ${step.StartTime.toISOString()}`);
            }
          }
        }
      }
    }
    nextToken = status.NextToken;
  } while (nextToken);
}

Failure modes and common mistakes