MCP Server LiteLLM — unified LLM proxy, provider routing, budget limits, streaming

LiteLLM client setup and model routing

LiteLLM exposes an OpenAI-compatible HTTP API, so you use the openai Node.js SDK with a custom baseURL. The critical difference from using OpenAI directly is the model name format. LiteLLM uses <provider>/<model> notation to route requests to the correct backend — the same proxy instance can hold credentials for OpenAI, Anthropic, AWS Bedrock, Azure, Cohere, and dozens of other providers simultaneously.

import OpenAI from 'openai';
import { z } from 'zod';

// LiteLLM is OpenAI-compatible — just change baseURL and apiKey
const litellm = new OpenAI({
  baseURL: process.env.LITELLM_BASE_URL ?? 'http://localhost:4000',  // default LiteLLM port
  apiKey:  process.env.LITELLM_API_KEY  ?? 'sk-1234',               // virtual key from LiteLLM UI
});

// Model names MUST include provider prefix — LiteLLM uses this for routing
// Omitting prefix causes LiteLLM to assume 'openai/' and fail if no OpenAI key is configured
const MODEL = process.env.LLM_MODEL ?? 'anthropic/claude-haiku-4-5-20251001';

// MCP tool — single-turn completion via LiteLLM
server.tool(
  'llm_complete',
  {
    prompt:      z.string().min(1).max(8000),
    max_tokens:  z.number().int().min(1).max(4096).optional().default(512),
    temperature: z.number().min(0).max(2).optional().default(0.7),
  },
  async ({ prompt, max_tokens, temperature }) => {
    const response = await litellm.chat.completions.create({
      model:       MODEL,
      messages:    [{ role: 'user', content: prompt }],
      max_tokens,
      temperature,
    });

    // LiteLLM response headers carry spend tracking — read via raw response
    const text = response.choices[0]?.message?.content ?? '';
    return {
      content: [{ type: 'text', text }],
    };
  }
);

// MCP tool — multi-turn chat via LiteLLM with spend header inspection
server.tool(
  'llm_chat',
  {
    messages: z.array(z.object({
      role:    z.enum(['user', 'assistant', 'system']),
      content: z.string().min(1),
    })).min(1).max(20),
    model: z.string().optional(),  // allow caller to specify provider/model
  },
  async ({ messages, model }) => {
    // Use withResponse() to access headers — needed for spend tracking
    const { data: completion, response } = await litellm.chat.completions
      .create({ model: model ?? MODEL, messages })
      .withResponse();

    const spend       = response.headers.get('x-litellm-spend');
    const modelCost   = response.headers.get('x-litellm-model-response-cost');
    const tokensLeft  = response.headers.get('x-ratelimit-remaining-tokens');

    const text = completion.choices[0]?.message?.content ?? '';
    return {
      content: [{
        type: 'text',
        text: JSON.stringify({
          reply:          text,
          spend_usd:      spend       ? parseFloat(spend)     : null,
          cost_usd:       modelCost   ? parseFloat(modelCost) : null,
          tokens_left:    tokensLeft  ? parseInt(tokensLeft)  : null,
        }),
      }],
    };
  }
);

LiteLLM model aliases (configured in the proxy's config.yaml) let you use short names like fast-model or smart-model in tool calls and remap them centrally without touching MCP server code. Use aliases in production so each deployment can tune cost/speed tradeoffs through LiteLLM config.

Budget exhaustion and rate limit error handling

LiteLLM's budget enforcement returns different error shapes depending on whether the limit is a hard budget cap on a virtual key versus a provider-side rate limit. Both surface as HTTP 429 but differ in error.type. MCP tools must distinguish them — a budget exhaustion should surface a helpful message to the agent, while a rate limit should be retried with backoff.

import OpenAI from 'openai';

// Wrap LiteLLM calls with budget-aware error handling
async function litellmCall(
  litellm: OpenAI,
  params: OpenAI.Chat.ChatCompletionCreateParamsNonStreaming
): Promise<string> {
  try {
    const completion = await litellm.chat.completions.create(params);
    return completion.choices[0]?.message?.content ?? '';
  } catch (err: any) {
    // LiteLLM budget exhaustion — NOT the same as OpenAI quota error
    if (err?.status === 429) {
      const errType = err?.error?.type ?? err?.error?.error?.type ?? '';

      if (errType === 'BudgetExceeded') {
        // Hard budget cap hit on this virtual key — do not retry
        throw new Error(
          `LiteLLM budget exhausted for key. ` +
          `Remaining budget: ${err?.error?.message ?? 'unknown'}. ` +
          `Contact your administrator to increase the budget limit.`
        );
      }

      if (errType === 'RateLimitError' || err?.code === 'rate_limit_exceeded') {
        // Provider-side rate limit — wait for retry-after header
        const retryAfter = err?.headers?.['retry-after'] ?? '60';
        throw new Error(
          `Rate limited by provider. Retry after ${retryAfter}s. ` +
          `Model: ${params.model}`
        );
      }
    }

    // AuthenticationError — usually wrong provider prefix or missing key
    if (err?.status === 401 || err?.status === 403) {
      throw new Error(
        `LiteLLM authentication failed for model '${params.model}'. ` +
        `Verify the provider prefix matches a configured model in LiteLLM config.yaml.`
      );
    }

    // Model not found — prefix correct but model name wrong
    if (err?.status === 404) {
      throw new Error(
        `Model '${params.model}' not found in LiteLLM router. ` +
        `Check config.yaml for available model names and their provider prefixes.`
      );
    }

    throw err;
  }
}

// MCP tool wrapping the error-aware helper
server.tool(
  'llm_safe_complete',
  {
    prompt:     z.string().min(1).max(8000),
    max_tokens: z.number().int().min(1).max(4096).optional().default(512),
  },
  async ({ prompt, max_tokens }) => {
    try {
      const text = await litellmCall(litellm, {
        model:    MODEL,
        messages: [{ role: 'user', content: prompt }],
        max_tokens,
      });
      return { content: [{ type: 'text', text }] };
    } catch (err: any) {
      return {
        content: [{ type: 'text', text: JSON.stringify({ error: err.message }) }],
        isError: true,
      };
    }
  }
);

Streaming completions from LiteLLM

LiteLLM supports OpenAI-compatible streaming. For MCP tools, streaming is most useful when the LLM response will be long — the tool can accumulate chunks and return the full text, or (for advanced use cases) emit intermediate progress. The key implementation detail is that usage is only present on the final chunk when you pass stream_options: { include_usage: true }.

// MCP tool — stream a LiteLLM completion and return full text
server.tool(
  'llm_stream',
  {
    prompt:     z.string().min(1).max(8000),
    max_tokens: z.number().int().min(1).max(8192).optional().default(1024),
  },
  async ({ prompt, max_tokens }) => {
    const stream = await litellm.chat.completions.create({
      model:          MODEL,
      messages:       [{ role: 'user', content: prompt }],
      max_tokens,
      stream:         true,
      stream_options: { include_usage: true },  // usage on final chunk
    });

    let fullText   = '';
    let usage: OpenAI.CompletionUsage | null = null;

    for await (const chunk of stream) {
      const delta = chunk.choices[0]?.delta?.content ?? '';
      fullText += delta;

      // usage is on the final chunk only when stream_options.include_usage = true
      if (chunk.usage) {
        usage = chunk.usage;
      }

      // Stop reason: 'stop' = natural end, 'length' = hit max_tokens
      const finishReason = chunk.choices[0]?.finish_reason;
      if (finishReason === 'length') {
        fullText += '\n[truncated — hit max_tokens limit]';
      }
    }

    return {
      content: [{
        type: 'text',
        text: JSON.stringify({
          text:              fullText,
          prompt_tokens:     usage?.prompt_tokens     ?? null,
          completion_tokens: usage?.completion_tokens ?? null,
          total_tokens:      usage?.total_tokens      ?? null,
        }),
      }],
    };
  }
);

Health probe and model availability check

LiteLLM exposes GET /health which pings each configured model backend and returns per-model status. Use this in your MCP server's health resource to surface which LLM providers are reachable without making a real inference call. LiteLLM also has GET /health/readiness (checks if the proxy process is up without pinging backends) and GET /health/liveliness (same). Use /health for detailed model status at startup; use /health/readiness for lightweight Kubernetes readiness probes.

// MCP resource — LiteLLM health check across all configured models
server.resource('litellm_health', 'litellm://health', async () => {
  const LITELLM_BASE = process.env.LITELLM_BASE_URL ?? 'http://localhost:4000';

  try {
    // GET /health pings each model backend — may take several seconds
    const res = await fetch(`${LITELLM_BASE}/health`, {
      headers: { Authorization: `Bearer ${process.env.LITELLM_API_KEY ?? ''}` },
      signal:  AbortSignal.timeout(15_000),  // allow up to 15s for all backends to respond
    });

    if (!res.ok) {
      return {
        contents: [{
          uri: 'litellm://health',
          text: JSON.stringify({ status: 'error', http_status: res.status }),
        }],
      };
    }

    const body = await res.json() as {
      healthy_endpoints:   Array<{ model: string; status: string }>;
      unhealthy_endpoints: Array<{ model: string; status: string; error: string }>;
      healthy_count:       number;
      unhealthy_count:     number;
    };

    const summary = {
      status:           body.unhealthy_count === 0 ? 'ok' : 'degraded',
      healthy_count:    body.healthy_count,
      unhealthy_count:  body.unhealthy_count,
      unhealthy_models: body.unhealthy_endpoints.map(e => ({
        model: e.model,
        error: e.error,
      })),
      checked_at: new Date().toISOString(),
    };

    return {
      contents: [{ uri: 'litellm://health', text: JSON.stringify(summary) }],
    };
  } catch (err: any) {
    return {
      contents: [{
        uri: 'litellm://health',
        text: JSON.stringify({ status: 'unreachable', message: err.message }),
      }],
    };
  }
});

The GET /models endpoint returns the full list of model IDs available to your virtual key. Call it at MCP server startup to validate that your configured LLM_MODEL is actually routable before the server begins accepting tool calls — a misconfigured model prefix causes silent failures that only surface at inference time.