MCP Server Hugging Face Inference API — pipelines, cold starts, dedicated endpoints, embeddings

Serverless Inference API — text generation and cold start handling

The Hugging Face Serverless Inference API hosts thousands of public models. Requests are billed per token and models are loaded on demand — if a model has not been called recently it will be in a cold state and return 503 until loaded. Free tier has strict rate limits (~300 requests/hour per token). Production MCP servers should either use a Pro/Enterprise HF account or switch to a Dedicated Endpoint once request volume justifies the hourly cost.

import { HfInference, InferenceOutputError } from '@huggingface/inference';
import { z } from 'zod';

const hf = new HfInference(process.env.HF_TOKEN);

// Cold-start-aware wrapper for serverless HF inference
async function hfWithColdStartRetry<T>(
  fn: () => Promise<T>,
  maxAttempts = 3
): Promise<T> {
  let lastErr: unknown;

  for (let attempt = 0; attempt < maxAttempts; attempt++) {
    try {
      return await fn();
    } catch (err: any) {
      lastErr = err;

      // HF cold-start: 503 with estimatedTime — must wait before retry
      // InferenceOutputError has .message that contains the JSON
      const message = err?.message ?? '';
      const coldStartMatch = message.match(/"estimated_time"\s*:\s*(\d+(?:\.\d+)?)/);

      if (coldStartMatch) {
        const waitSeconds = Math.min(parseFloat(coldStartMatch[1]) + 2, 120);
        // Wait for estimated_time + 2s buffer before retrying
        await new Promise(resolve => setTimeout(resolve, waitSeconds * 1000));
        continue;
      }

      // 503 without estimated_time — transient server error, short backoff
      if (err?.httpStatus === 503 || message.includes('503')) {
        await new Promise(resolve => setTimeout(resolve, (attempt + 1) * 5000));
        continue;
      }

      // 429 rate limit — respect Retry-After header or use backoff
      if (err?.httpStatus === 429) {
        await new Promise(resolve => setTimeout(resolve, (attempt + 1) * 10_000));
        continue;
      }

      // Non-retriable error — rethrow immediately
      throw err;
    }
  }

  throw lastErr;
}

// MCP tool — text generation with cold-start retry
server.tool(
  'hf_generate',
  {
    model:      z.string().min(1).default('mistralai/Mistral-7B-Instruct-v0.2'),
    prompt:     z.string().min(1).max(4000),
    max_tokens: z.number().int().min(1).max(2048).optional().default(256),
    temperature: z.number().min(0).max(2).optional().default(0.7),
  },
  async ({ model, prompt, max_tokens, temperature }) => {
    const result = await hfWithColdStartRetry(() =>
      hf.textGeneration({
        model,
        inputs: prompt,
        parameters: {
          max_new_tokens: max_tokens,
          temperature,
          return_full_text: false,  // return only new tokens, not prompt + generated
          do_sample: temperature > 0,
        },
      })
    );

    return {
      content: [{ type: 'text', text: result.generated_text }],
    };
  }
);

Set return_full_text: false — the default is true, which prepends your prompt to the generated output. For MCP tools returning completion text to an agent, false is almost always what you want. For instruction-tuned models (Mistral-Instruct, Llama-3-Instruct), also wrap your prompt in the model's chat template — the Inference API does not apply chat templates automatically for textGeneration; use hf.chatCompletion() if the model supports the Messages API.

Embeddings for RAG — feature extraction

Embedding models on HF use the feature-extraction pipeline task. The output is a nested array: for a single string input it returns number[][] (one embedding per token) or just number[] depending on whether pooling is applied. Most sentence-transformer models return a flat number[] when called through the Inference API — but zero-shot or raw transformer models return a 2D array; you must call output[0] to get the embedding for the [CLS] token, or average the token embeddings yourself.

// MCP tool — single-text embedding via HF Inference API
server.tool(
  'hf_embed',
  {
    text:  z.string().min(1).max(8192),
    model: z.string().min(1).default('sentence-transformers/all-MiniLM-L6-v2'),
  },
  async ({ text, model }) => {
    const raw = await hfWithColdStartRetry(() =>
      hf.featureExtraction({ model, inputs: text })
    );

    // Normalize output shape — sentence-transformers return float[]
    // raw transformer models return float[][] (tokens × dims)
    let embedding: number[];
    if (Array.isArray(raw[0])) {
      // 2D output — mean pool token embeddings (exclude padding)
      const matrix = raw as number[][];
      const dims = matrix[0].length;
      embedding = Array(dims).fill(0).map((_, d) =>
        matrix.reduce((sum, tok) => sum + tok[d], 0) / matrix.length
      );
    } else {
      embedding = raw as number[];
    }

    return {
      content: [{
        type: 'text',
        text: JSON.stringify({
          embedding,
          dimensions: embedding.length,
          model,
        }),
      }],
    };
  }
);

// MCP tool — batch embeddings (up to 64 texts per request on serverless)
server.tool(
  'hf_embed_batch',
  {
    texts: z.array(z.string().min(1).max(512)).min(1).max(64),
    model: z.string().min(1).default('sentence-transformers/all-MiniLM-L6-v2'),
  },
  async ({ texts, model }) => {
    const raw = await hfWithColdStartRetry(() =>
      hf.featureExtraction({ model, inputs: texts })
    );

    // Batch input returns float[][] — one embedding per input text
    const embeddings = raw as number[][];
    return {
      content: [{
        type: 'text',
        text: JSON.stringify({
          embeddings,
          count:      embeddings.length,
          dimensions: embeddings[0]?.length ?? 0,
          model,
        }),
      }],
    };
  }
);

Dedicated Endpoints — no cold starts, fixed hourly cost

HF Dedicated Endpoints run a single model on a reserved GPU instance. The endpoint URL is assigned at creation time and never changes. Unlike serverless, dedicated endpoints have no cold start delay and no shared rate limits — throughput is bounded only by the GPU. They expose a /health HTTP endpoint that returns 200 when the model is loaded and ready.

import { HfInference } from '@huggingface/inference';

// Dedicated endpoint — use hf.endpoint() to wrap the custom URL
const ENDPOINT_URL = process.env.HF_ENDPOINT_URL!;  // e.g. https://abc123.us-east-1.aws.endpoints.huggingface.cloud
const hfEndpoint   = hf.endpoint(ENDPOINT_URL);

// MCP tool — text generation via Dedicated Endpoint (no cold-start retry needed)
server.tool(
  'hf_endpoint_generate',
  {
    prompt:      z.string().min(1).max(8000),
    max_tokens:  z.number().int().min(1).max(4096).optional().default(512),
    temperature: z.number().min(0).max(2).optional().default(0.7),
  },
  async ({ prompt, max_tokens, temperature }) => {
    // Dedicated endpoints do not cold-start — no retry needed
    const result = await hfEndpoint.textGeneration({
      inputs: prompt,
      parameters: {
        max_new_tokens:  max_tokens,
        temperature,
        return_full_text: false,
        do_sample:       temperature > 0,
      },
    });

    return { content: [{ type: 'text', text: result.generated_text }] };
  }
);

// MCP resource — Dedicated Endpoint health probe
server.resource('hf_endpoint_health', 'hf://endpoint/health', async () => {
  try {
    // Dedicated Endpoints expose GET /health — serverless does NOT
    const res = await fetch(`${ENDPOINT_URL}/health`, {
      headers: { Authorization: `Bearer ${process.env.HF_TOKEN}` },
      signal:  AbortSignal.timeout(10_000),
    });

    const status = res.status === 200 ? 'ok' : 'degraded';
    const body   = res.ok ? await res.json().catch(() => null) : null;

    return {
      contents: [{
        uri: 'hf://endpoint/health',
        text: JSON.stringify({
          status,
          http_status: res.status,
          endpoint:    ENDPOINT_URL,
          model_info:  body,
          checked_at:  new Date().toISOString(),
        }),
      }],
    };
  } catch (err: any) {
    return {
      contents: [{
        uri: 'hf://endpoint/health',
        text: JSON.stringify({ status: 'unreachable', message: err.message }),
      }],
    };
  }
});

For MCP servers in production, prefer Dedicated Endpoints over the Serverless API when your tools are called frequently (more than ~100 times/hour) or when cold-start latency is unacceptable. The break-even point varies by model size: a 7B model endpoint costs roughly $0.50/hour — if your MCP server makes 1,000+ inferences per hour, dedicated is cheaper and more predictable than paying per-token on serverless.