AWS Rekognition Text Detection (OCR) in MCP Servers — DetectText, LINE/WORD Blocks, Reading Order

Basic tool implementation

import { RekognitionClient, DetectTextCommand } from '@aws-sdk/client-rekognition';
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
import { z } from 'zod';

const rekognition = new RekognitionClient({ region: 'us-east-1' });
const server = new McpServer({ name: 'ocr-tools', version: '1.0.0' });

server.tool('extract_text_from_image', {
  s3_bucket: z.string(),
  s3_key: z.string(),
  min_confidence: z.number().min(0).max(100).default(80),
  reading_order: z.boolean().default(true),
}, async ({ s3_bucket, s3_key, min_confidence, reading_order }) => {
  const { TextDetections = [] } = await rekognition.send(new DetectTextCommand({
    Image: { S3Object: { Bucket: s3_bucket, Name: s3_key } },
    Filters: {
      WordFilter: { MinConfidence: min_confidence },
    },
  }));

  // Extract only LINE blocks (each already contains the full line text)
  let lines = TextDetections
    .filter(d => d.Type === 'LINE' && (d.Confidence ?? 0) >= min_confidence)
    .map(d => ({
      text: d.DetectedText ?? '',
      confidence: Math.round((d.Confidence ?? 0) * 10) / 10,
      bounding_box: d.Geometry?.BoundingBox,
      id: d.Id,
    }));

  if (reading_order) {
    // Sort top-to-bottom, left-to-right to reconstruct reading order
    lines = lines.sort((a, b) => {
      const topDiff = (a.bounding_box?.Top ?? 0) - (b.bounding_box?.Top ?? 0);
      if (Math.abs(topDiff) > 0.02) return topDiff; // different rows
      return (a.bounding_box?.Left ?? 0) - (b.bounding_box?.Left ?? 0);
    });
  }

  const fullText = lines.map(l => l.text).join('\n');

  return {
    content: [{
      type: 'text',
      text: JSON.stringify({
        full_text: fullText,
        line_count: lines.length,
        lines,
      }, null, 2),
    }],
  };
});

The 0.02 tolerance in the row-sort groups text at approximately the same vertical position on the same "line" before sorting left-to-right. Without this tolerance, fractional-pixel differences in top position would scatter words from the same visual row into separate sort groups. Tune it based on the expected line height of your target images — for dense tables use 0.01, for large-print signage use 0.05.

Block structure — LINE and WORD relationship

The TextDetections array mixes LINE and WORD entries. Each WORD's ParentId references the Id of its parent LINE. This graph structure lets you pick the right level of granularity:

// Full block structure example
// [ { Id: 0, Type: 'LINE', DetectedText: 'STOP', ParentId: undefined },
//   { Id: 1, Type: 'WORD', DetectedText: 'STOP', ParentId: 0 },
//   { Id: 2, Type: 'LINE', DetectedText: 'No entry beyond this point', ParentId: undefined },
//   { Id: 3, Type: 'WORD', DetectedText: 'No', ParentId: 2 },
//   { Id: 4, Type: 'WORD', DetectedText: 'entry', ParentId: 2 },
//   ... ]

// Get all words in a specific line by ParentId
function getWordsInLine(detections: any[], lineId: number) {
  return detections
    .filter(d => d.Type === 'WORD' && d.ParentId === lineId)
    .sort((a, b) => (a.Geometry?.BoundingBox?.Left ?? 0) - (b.Geometry?.BoundingBox?.Left ?? 0));
}

// Build word-level index for per-word confidence inspection
function buildWordIndex(detections: any[]) {
  const wordsByLine = new Map<number, typeof detections>();
  for (const d of detections) {
    if (d.Type === 'WORD' && d.ParentId !== undefined) {
      if (!wordsByLine.has(d.ParentId)) wordsByLine.set(d.ParentId, []);
      wordsByLine.get(d.ParentId)!.push(d);
    }
  }
  return wordsByLine;
}

// Use WORD-level blocks when you need per-word confidence scores
// (useful for redacting low-confidence words from a transcription)
const detections = /* ... */;
const linesWithWordConfidence = detections
  .filter(d => d.Type === 'LINE')
  .map(line => {
    const words = getWordsInLine(detections, line.Id);
    return {
      line_text: line.DetectedText,
      words: words.map(w => ({
        word: w.DetectedText,
        confidence: w.Confidence,
      })),
      min_word_confidence: Math.min(...words.map(w => w.Confidence ?? 0)),
    };
  });

The LINE-level Confidence is a composite score. If a line contains both high-confidence and low-confidence words, the LINE confidence is a weighted aggregate. For applications where every word must be reliable (e.g., reading serial numbers), inspect the WORD-level scores and reject lines where any WORD falls below your threshold.

BoundingBox vs Polygon for rotated text

Each detection has two geometry representations:

// BoundingBox: axis-aligned rectangle (always horizontal)
// Good for: standard horizontal text, layout sorting, quick overlap checks
{
  "BoundingBox": {
    "Left": 0.15,   // left edge x as fraction of image width
    "Top": 0.42,    // top edge y as fraction of image height
    "Width": 0.60,
    "Height": 0.08
  }
}

// Polygon: 4-point quadrilateral that hugs rotated or curved text
// Good for: angled signs, text on tilted documents, perspective-distorted labels
{
  "Polygon": [
    { "X": 0.14, "Y": 0.44 },   // top-left corner
    { "X": 0.75, "Y": 0.40 },   // top-right corner
    { "X": 0.76, "Y": 0.52 },   // bottom-right corner
    { "X": 0.15, "Y": 0.56 }    // bottom-left corner
  ]
}

// Convert fractional polygon to pixel coordinates
function polygonToPixels(polygon: Array<{X: number; Y: number}>, w: number, h: number) {
  return polygon.map(pt => ({ x: Math.round(pt.X * w), y: Math.round(pt.Y * h) }));
}

// Compute rotation angle from polygon (useful for deskewing)
function estimateTextAngle(polygon: Array<{X: number; Y: number}>): number {
  const [tl, tr] = polygon; // first two points = top edge
  const dx = tr.X - tl.X;
  const dy = tr.Y - tl.Y;
  return Math.atan2(dy, dx) * (180 / Math.PI);
}

For reading-order sorting of rotated text, sort by the top-left polygon point's Y coordinate rather than BoundingBox.Top. The BoundingBox.Top of a 45° rotated text block will misplace it relative to horizontal neighbours because the axis-aligned box inflates in both dimensions.

Regions of Interest

For fixed-layout images (e.g., camera stills where the licence plate is always in the lower-right quadrant), restricting detection to a region reduces noise and improves throughput:

// RegionsOfInterest constrains detection to specified image areas
const { TextDetections = [] } = await rekognition.send(new DetectTextCommand({
  Image: { S3Object: { Bucket: 'traffic-cam-frames', Name: 'frame-00123.jpg' } },
  Filters: {
    RegionsOfInterest: [
      {
        BoundingBox: {
          Left: 0.60,   // right 40% of image
          Top: 0.65,    // bottom 35% of image
          Width: 0.40,
          Height: 0.35,
        },
      },
    ],
    WordFilter: { MinConfidence: 85 },
  },
}));

// Multiple regions — e.g., header and footer of a fixed-format form
const formRegions = {
  Filters: {
    RegionsOfInterest: [
      { BoundingBox: { Left: 0, Top: 0, Width: 1, Height: 0.15 } },     // header
      { BoundingBox: { Left: 0, Top: 0.85, Width: 1, Height: 0.15 } },  // footer
    ],
  },
};

RegionsOfInterest is applied server-side — text outside the specified regions is not evaluated. This reduces API latency for large images and eliminates false detections from background textures or unrelated text in the frame.

Practical limits

When you need more than 100 words, structured document parsing (multi-column PDFs, forms with dozens of fields), or non-Latin script support, switch to AWS Textract. Textract handles unlimited words, tables, key-value pairs, and complex document layouts but requires JPEG or PDF input and is priced higher per page.

Async video text detection

import {
  StartTextDetectionCommand,
  GetTextDetectionCommand,
} from '@aws-sdk/client-rekognition';

// Two-tool MCP pattern: start + poll
server.tool('start_video_text_detection', {
  s3_bucket: z.string(),
  s3_key: z.string(),
  min_confidence: z.number().default(80),
}, async ({ s3_bucket, s3_key, min_confidence }) => {
  const { JobId } = await rekognition.send(new StartTextDetectionCommand({
    Video: { S3Object: { Bucket: s3_bucket, Name: s3_key } },
    Filters: { WordFilter: { MinConfidence: min_confidence } },
  }));
  return { content: [{ type: 'text', text: JSON.stringify({ job_id: JobId }) }] };
});

server.tool('get_video_text_results', {
  job_id: z.string(),
}, async ({ job_id }) => {
  const allDetections: any[] = [];
  let nextToken: string | undefined;

  do {
    const result = await rekognition.send(new GetTextDetectionCommand({
      JobId: job_id,
      MaxResults: 1000,
      NextToken: nextToken,
    }));

    if (result.JobStatus === 'FAILED') {
      return { content: [{ type: 'text', text: JSON.stringify({ error: result.StatusMessage }) }] };
    }
    if (result.JobStatus !== 'SUCCEEDED') {
      return { content: [{ type: 'text', text: JSON.stringify({ status: 'in_progress' }) }] };
    }

    allDetections.push(...(result.TextDetections ?? []));
    nextToken = result.NextToken;
  } while (nextToken);

  // Video: each detection has Timestamp (ms). Extract unique text per timestamp.
  const linesByTimestamp = new Map<number, string[]>();
  for (const d of allDetections) {
    if (d.TextDetection?.Type === 'LINE') {
      const ts = d.Timestamp ?? 0;
      if (!linesByTimestamp.has(ts)) linesByTimestamp.set(ts, []);
      linesByTimestamp.get(ts)!.push(d.TextDetection.DetectedText ?? '');
    }
  }

  const timeline = Array.from(linesByTimestamp.entries())
    .sort(([a], [b]) => a - b)
    .map(([ts, lines]) => ({ timestamp_ms: ts, text: lines.join(' | ') }));

  return { content: [{ type: 'text', text: JSON.stringify({ timeline, frame_count: timeline.length }) }] };
});

Video text detection is useful for caption extraction from footage, reading scoreboards or ticker tapes, or indexing text that appears briefly on screen. The Timestamp (milliseconds from video start) lets you correlate text appearances with specific video moments. Text that persists across frames will appear at multiple timestamps — deduplicate consecutive identical strings if you want a list of unique text appearances.

Choosing between Rekognition DetectText and Textract