MCP Server vLLM — local model serving, OpenAI-compatible API, chat template, sampling params

vLLM client setup and model name discovery

vLLM's HTTP server is OpenAI-compatible, running by default on port 8000. The apiKey field is ignored by vLLM but required by the openai SDK — pass any non-empty string like 'EMPTY' or 'token-abc'. The critical startup step is discovering the model name — it may differ from the HuggingFace ID if the operator set --served-model-name.

import OpenAI from 'openai';
import { z } from 'zod';

const VLLM_BASE_URL = process.env.VLLM_BASE_URL ?? 'http://localhost:8000/v1';

const vllm = new OpenAI({
  baseURL: VLLM_BASE_URL,
  apiKey:  process.env.VLLM_API_KEY ?? 'EMPTY',  // vLLM ignores the key but SDK requires it
});

// Discover the model name at startup — do not hardcode
// vLLM's --served-model-name alias may differ from the HuggingFace model ID
let servedModelName: string | undefined;

async function getServedModel(): Promise<string> {
  if (servedModelName) return servedModelName;

  const models = await vllm.models.list();
  const first  = models.data[0];
  if (!first) throw new Error('vLLM returned no models from GET /v1/models');

  servedModelName = first.id;
  return servedModelName;
}

// MCP tool — chat completion via vLLM (instruction-tuned models only)
server.tool(
  'vllm_chat',
  {
    messages: z.array(z.object({
      role:    z.enum(['user', 'assistant', 'system']),
      content: z.string().min(1),
    })).min(1).max(20),
    max_tokens:  z.number().int().min(1).max(8192).optional().default(512),
    temperature: z.number().min(0).max(2).optional().default(0.7),
  },
  async ({ messages, max_tokens, temperature }) => {
    const model = await getServedModel();

    const completion = await vllm.chat.completions.create({
      model,
      messages,
      max_tokens,
      temperature,
    });

    const text = completion.choices[0]?.message?.content ?? '';
    return { content: [{ type: 'text', text }] };
  }
);

// MCP tool — raw completion via vLLM (works with base models too)
server.tool(
  'vllm_complete',
  {
    prompt:      z.string().min(1).max(32768),
    max_tokens:  z.number().int().min(1).max(8192).optional().default(512),
    temperature: z.number().min(0).max(2).optional().default(0.7),
    stop:        z.array(z.string()).max(4).optional(),
  },
  async ({ prompt, max_tokens, temperature, stop }) => {
    const model = await getServedModel();

    // /v1/completions — works for both base and instruct models
    const completion = await vllm.completions.create({
      model,
      prompt,
      max_tokens,
      temperature,
      stop: stop ?? [],
    });

    const text = completion.choices[0]?.text ?? '';
    return { content: [{ type: 'text', text }] };
  }
);

When you deploy vLLM with multiple model aliases (--served-model-name can be specified multiple times), /v1/models returns all aliases. Query and select the right one by name rather than by position — the order is not guaranteed across vLLM restarts.

vLLM-specific sampling parameters via extra_body

vLLM extends the OpenAI API with additional sampling controls. These are not recognized by the OpenAI SDK's TypeScript types, so they must be passed through extra_body — a catch-all dict that the SDK forwards verbatim. The most commonly needed extensions are skip_special_tokens (strip <|im_end|> tokens from output), guided_json (constrained decoding to a JSON schema), and stop_token_ids (stop on specific token IDs rather than strings).

// MCP tool — structured JSON output via vLLM guided decoding
server.tool(
  'vllm_json_extract',
  {
    text:   z.string().min(1).max(8000),
    schema: z.record(z.unknown()),  // JSON Schema object for the desired output
  },
  async ({ text, schema }) => {
    const model = await getServedModel();

    const completion = await vllm.chat.completions.create({
      model,
      messages: [{
        role:    'user',
        content: `Extract information from the following text as JSON matching the provided schema.\n\nText:\n${text}`,
      }],
      max_tokens: 1024,
      temperature: 0,  // deterministic for structured extraction
      // vLLM-specific params in extra_body — SDK forwards these verbatim
      extra_body: {
        guided_json:           schema,          // constrain output to this JSON Schema
        guided_decoding_backend: 'outlines',   // 'outlines' or 'lm-format-enforcer'
        skip_special_tokens:   true,           // strip chat template end tokens from output
      } as Record<string, unknown>,
    });

    const raw = completion.choices[0]?.message?.content ?? '{}';
    try {
      const parsed = JSON.parse(raw);
      return { content: [{ type: 'text', text: JSON.stringify(parsed) }] };
    } catch {
      return {
        content: [{ type: 'text', text: JSON.stringify({ error: 'parse_failed', raw }) }],
        isError: true,
      };
    }
  }
);

// MCP tool — best-of-N sampling for higher quality output
server.tool(
  'vllm_best_of',
  {
    prompt:    z.string().min(1).max(8000),
    n:         z.number().int().min(2).max(8).optional().default(3),
    max_tokens: z.number().int().min(1).max(2048).optional().default(256),
  },
  async ({ prompt, n, max_tokens }) => {
    const model = await getServedModel();

    const completion = await vllm.completions.create({
      model,
      prompt,
      max_tokens,
      n,  // generate n candidates in one request
      extra_body: {
        best_of:             n,    // vLLM samples best_of and returns n best
        skip_special_tokens: true,
      } as Record<string, unknown>,
    });

    // Return all candidates for the agent to pick from
    const candidates = completion.choices.map((c, i) => ({
      index: i,
      text:  c.text,
      logprob: c.logprobs?.token_logprobs?.reduce((s, v) => s + (v ?? 0), 0) ?? null,
    }));

    return {
      content: [{ type: 'text', text: JSON.stringify({ candidates }) }],
    };
  }
);

Streaming and token-by-token output

vLLM streaming is standard OpenAI SSE format. For MCP tools, streaming is most useful when you want to return partial results as the model generates — for example, a tool that writes a long document. In practice most MCP clients do not display streaming progress, so accumulating the full response is safer. Always check finish_reason — 'length' means the model hit max_tokens before completing its thought.

// MCP tool — streaming completion, returns accumulated text
server.tool(
  'vllm_stream',
  {
    prompt:      z.string().min(1).max(16384),
    max_tokens:  z.number().int().min(1).max(8192).optional().default(1024),
    temperature: z.number().min(0).max(2).optional().default(0.7),
  },
  async ({ prompt, max_tokens, temperature }) => {
    const model  = await getServedModel();
    const stream = await vllm.completions.create({
      model,
      prompt,
      max_tokens,
      temperature,
      stream: true,
      extra_body: { skip_special_tokens: true } as Record<string, unknown>,
    });

    let fullText    = '';
    let finishReason: string | null = null;
    let totalTokens = 0;

    for await (const chunk of stream) {
      fullText    += chunk.choices[0]?.text ?? '';
      finishReason = chunk.choices[0]?.finish_reason ?? finishReason;
      totalTokens  = chunk.usage?.total_tokens ?? totalTokens;
    }

    return {
      content: [{
        type: 'text',
        text: JSON.stringify({
          text:          fullText,
          finish_reason: finishReason,
          total_tokens:  totalTokens,
          truncated:     finishReason === 'length',
        }),
      }],
    };
  }
);

Health probe and startup validation

vLLM exposes GET /health which returns HTTP 200 once the model is fully loaded into GPU memory and ready to serve requests, and HTTP 503 during the loading phase. This can take several minutes for large models. MCP servers that start before vLLM is ready should retry the health endpoint with backoff rather than fail-fast — vLLM's startup time depends on model size and GPU bandwidth, not a bug.

// MCP resource — vLLM health probe
server.resource('vllm_health', 'vllm://health', async () => {
  const base = VLLM_BASE_URL.replace('/v1', '');  // health is at root, not /v1

  try {
    const res = await fetch(`${base}/health`, {
      signal: AbortSignal.timeout(5_000),
    });

    if (res.status === 200) {
      // Also fetch model info for context
      const models = await vllm.models.list().catch(() => null);
      const modelId = models?.data[0]?.id ?? 'unknown';

      return {
        contents: [{
          uri: 'vllm://health',
          text: JSON.stringify({
            status:     'ok',
            model:      modelId,
            checked_at: new Date().toISOString(),
          }),
        }],
      };
    }

    return {
      contents: [{
        uri: 'vllm://health',
        text: JSON.stringify({
          status:      res.status === 503 ? 'loading' : 'error',
          http_status: res.status,
          checked_at:  new Date().toISOString(),
        }),
      }],
    };
  } catch (err: any) {
    return {
      contents: [{
        uri: 'vllm://health',
        text: JSON.stringify({ status: 'unreachable', message: err.message }),
      }],
    };
  }
});

// Startup validation — wait for vLLM to be ready before accepting tool calls
async function waitForVllm(maxWaitMs = 300_000): Promise<void> {
  const base    = VLLM_BASE_URL.replace('/v1', '');
  const start   = Date.now();
  let   attempt = 0;

  while (Date.now() - start < maxWaitMs) {
    try {
      const res = await fetch(`${base}/health`, { signal: AbortSignal.timeout(5_000) });
      if (res.status === 200) return;
    } catch {}

    // Backoff: 5s, 10s, 15s, max 30s between checks
    const delay = Math.min(5_000 * (attempt + 1), 30_000);
    await new Promise(r => setTimeout(r, delay));
    attempt++;
  }

  throw new Error(`vLLM did not become ready within ${maxWaitMs / 1000}s`);
}

Call await waitForVllm() in your MCP server's initialization code before registering tools. An MCP server that registers tools before vLLM is ready will surface ECONNREFUSED errors on the first tool call, which agents interpret as a broken tool rather than a transient startup condition.