MCP Server X-Ray Annotations and Metadata — indexing tool names, filter expressions, groups

Annotations vs metadata: the indexing boundary

// Annotations: indexed, queryable in filter expressions
// - Max 50 annotations per trace (segment + all subsegments combined)
// - Key: alphanumeric + underscores only, case-sensitive, max 500 chars
// - Value: string (max 250 chars), number, or boolean — nothing else
// - Used for: tool name, client id, tenant id, success flag, version

// Metadata: not indexed, not queryable
// - No size limit (bounded only by trace storage quota: 500KB per trace)
// - Key: any string
// - Value: any JSON-serializable value (objects, arrays, null)
// - Used for: full request args, response body preview, stack traces, timing breakdown

import AWSXRay from "aws-xray-sdk-node";

async function runTool(toolName: string, args: unknown, fn: () => Promise) {
  const segment = AWSXRay.resolveSegment();
  const sub = segment?.addNewSubsegment(`tool:${toolName}`);
  if (!sub) return fn(); // no trace context — run unwrapped

  // ANNOTATIONS — index these for filter expressions
  sub.addAnnotation("tool", toolName);                    // string, indexed
  sub.addAnnotation("tool_version", "1.0");              // string, indexed
  // sub.addAnnotation("args", JSON.stringify(args));    // WRONG: too long, fails silently at 250 chars

  const start = Date.now();
  try {
    const result = await fn();
    const duration = Date.now() - start;

    sub.addAnnotation("success", true);                  // boolean, indexed
    sub.addAnnotation("duration_bucket",                 // bucketed for histogram-style queries
      duration < 100 ? "fast" : duration < 1000 ? "normal" : "slow"
    );

    // METADATA — store full details for debugging
    sub.addMetadata("input", args);                      // full args object, not indexed
    sub.addMetadata("duration_ms", duration);            // raw number
    sub.addMetadata("result_preview",
      JSON.stringify(result).slice(0, 500)               // first 500 chars
    );

    sub.close();
    return result;
  } catch (err) {
    const duration = Date.now() - start;

    sub.addAnnotation("success", false);
    sub.addAnnotation("error_type",
      (err as Error).constructor.name.slice(0, 50)      // "ValidationError", "TimeoutError", etc.
    );
    sub.addMetadata("error", {
      message: (err as Error).message,
      stack: (err as Error).stack?.slice(0, 2000),      // cap stack trace length
      input: args,
    });
    sub.addError(err as Error);
    sub.close();
    throw err;
  }
}

Indexing tenant and client identity

MCP servers serving multiple clients or tenants can tag traces with client identity for per-tenant performance analysis. Extract the client ID from the MCP client info or from an auth header.

import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
import AWSXRay from "aws-xray-sdk-node";

// Annotate the root segment with client identity at connection time
function annotateSession(server: McpServer, segment: AWSXRay.Segment | null) {
  server.server.oninitialized = () => {
    const clientInfo = server.server.getClientVersion();
    // clientInfo: { name: "claude-desktop", version: "3.1.0" }
    segment?.addAnnotation("client_name",
      (clientInfo?.name ?? "unknown").slice(0, 50).replace(/[^a-zA-Z0-9_-]/g, "_")
    );
    segment?.addAnnotation("client_version",
      (clientInfo?.version ?? "unknown").slice(0, 20)
    );
  };
}

// For authenticated MCP servers: extract tenant from JWT or API key
function annotateFromAuth(segment: AWSXRay.Segment | null, authHeader: string | undefined) {
  if (!authHeader) {
    segment?.addAnnotation("tenant", "anonymous");
    return;
  }
  try {
    // Decode JWT without verification (already verified by middleware)
    const payload = JSON.parse(
      Buffer.from(authHeader.replace("Bearer ", "").split(".")[1], "base64").toString()
    );
    segment?.addAnnotation("tenant", (payload.sub ?? "unknown").slice(0, 100));
    segment?.addAnnotation("plan", (payload["custom:plan"] ?? "free").slice(0, 20));
  } catch {
    segment?.addAnnotation("tenant", "parse_error");
  }
}

Filter expressions for MCP tool analysis

Filter expressions query the annotation index to find specific traces in the X-Ray console or via the API. Write them against the annotations you added to your tool subsegments.

// X-Ray filter expression syntax:
// annotation.<key> = "value"          exact string match
// annotation.<key> != "value"         not equal
// annotation.<key> = true             boolean match
// annotation.<key> > 500              numeric comparison (for number annotations)
// service("name")                      filter by service name
// duration > 2                         trace total duration in seconds
// error = true                         traces where any segment has error flag
// fault = true                         traces where any segment has fault flag (5xx)
// responsetime > 3                     total response time in seconds

// Useful filter expressions for MCP server debugging:

// 1. All failures for a specific tool:
// annotation.tool = "search_documents" AND annotation.success = false

// 2. Slow traces for any tool (duration bucket = slow):
// annotation.duration_bucket = "slow"

// 3. All traces from a specific client:
// annotation.client_name = "claude_desktop"

// 4. Errors from a specific tenant on a specific tool:
// annotation.tenant = "acme-corp" AND annotation.tool = "create_record" AND fault = true

// 5. Traces with a specific error type:
// annotation.error_type = "ValidationError"

// 6. Cross-service: slow DynamoDB calls within tool traces:
// service("DynamoDB") AND duration > 0.5
// (service() matches X-Ray segment names — DynamoDB auto-segments use "DynamoDB")

// 7. Find all cold start invocations (Lambda):
// annotation.cold_start = true AND service("my-mcp-server")

X-Ray groups: segmented service maps and CloudWatch metrics

X-Ray groups are saved filter expressions that create isolated service maps and emit CloudWatch metrics. Create groups to get per-tool-category error rate metrics that you can alarm on.

// AWS CDK: create X-Ray groups for MCP server monitoring
import * as xray from "aws-cdk-lib/aws-xray";
import * as cloudwatch from "aws-cdk-lib/aws-cloudwatch";

// Group for tool failures — any tool call that set success=false
const toolErrorsGroup = new xray.CfnGroup(this, "McpToolErrorsGroup", {
  groupName: "mcp-tool-errors",
  filterExpression: "annotation.success = false AND service(\"mcp-server\")",
  insightsConfiguration: {
    insightsEnabled: true,   // enables anomaly detection on this group
    notificationsEnabled: true,
  },
});

// Group for slow tool calls
const slowToolsGroup = new xray.CfnGroup(this, "McpSlowToolsGroup", {
  groupName: "mcp-slow-tools",
  filterExpression: "annotation.duration_bucket = \"slow\" AND service(\"mcp-server\")",
  insightsConfiguration: { insightsEnabled: false }, // no anomaly detection needed
});

// X-Ray groups emit CloudWatch metrics under the "AWS/XRay" namespace:
// - ApproximateTraceCount: number of traces matching the filter per minute
// - ThrottleCount, ErrorCount, FaultCount: counts by response type
// Dimensions: { GroupName: "mcp-tool-errors" }

// Alarm: error rate > 5% of tool calls in a 5-minute window
const errorCountMetric = new cloudwatch.Metric({
  namespace: "AWS/XRay",
  metricName: "ErrorCount",
  dimensionsMap: { GroupName: "mcp-tool-errors" },
  period: cdk.Duration.minutes(5),
  statistic: "Sum",
});

new cloudwatch.Alarm(this, "McpToolErrorAlarm", {
  metric: errorCountMetric,
  threshold: 10,              // 10 errors in 5 minutes
  evaluationPeriods: 1,
  alarmDescription: "MCP tool error count exceeded threshold",
});

// AWS CLI: create a group without CDK
aws xray create-group \
  --group-name mcp-search-tool \
  --filter-expression 'annotation.tool = "search_documents"' \
  --insights-configuration InsightsEnabled=true,NotificationsEnabled=false \
  --region us-east-1

Annotation value sanitization

Annotation keys and values must meet X-Ray constraints. Keys that fail validation are silently dropped — the SDK does not throw an error. Sanitize before adding.

// Safe annotation helper — prevents silent drops
function safeAnnotation(
  sub: AWSXRay.Subsegment | AWSXRay.Segment | null,
  key: string,
  value: string | number | boolean
): void {
  if (!sub) return;

  // Key: alphanumeric + underscores only
  const safeKey = key.replace(/[^a-zA-Z0-9_]/g, "_").slice(0, 499);

  if (typeof value === "string") {
    // Value: max 250 chars — truncate rather than drop
    sub.addAnnotation(safeKey, value.slice(0, 249));
  } else {
    sub.addAnnotation(safeKey, value);
  }
}

// Usage:
safeAnnotation(sub, "tool", toolName);
safeAnnotation(sub, "client_name", clientInfo?.name ?? "unknown");
safeAnnotation(sub, "tenant_id", tenantId);
safeAnnotation(sub, "success", !hadError);

Common failure modes