Anthropic Messages API

# InferenceClass for a single-H100 Nebius shape, serving Qwen3-8B.
#
# Nebius sizes nodes by platform + preset rather than an instance type; the
# preset determines the GPU, vCPU, and memory shape of each node. The devices
# block describes what a node of this class has, DRA-style - used by the
# scheduler to match models to clusters, and to form DRA ResourceClaims for
# claim: DRA devices. An 8B model needs a fraction of an H100, so this shape has
# ample headroom; a single L40S (gpu-l40s-a) is the economical alternative.
apiVersion: modelplane.ai/v1alpha1
kind: InferenceClass
metadata:
  name: h100-1x
spec:
  description: "Nebius gpu-h100-sxm, 1x NVIDIA H100 80GB"
  provisioning:
    provider: Nebius
    nebius:
      platform: gpu-h100-sxm
      preset: 1gpu-16vcpu-200gb
      diskSizeGb: 200
      driversPreset: cuda13.0
      accelerator:
        type: nvidia-h100
        count: 1
  devices:
  - name: gpu
    claim: DRA
    driver: gpu.nvidia.com
    deviceClassName: gpu.nvidia.com
    count: 1
    attributes:
      architecture: { string: Hopper }
      cudaComputeCapability: { version: "9.0.0" }
    capacity:
      # The H100 80GB's real usable VRAM, what the NVIDIA DRA driver reports,
      # not its nominal 80GB. A nodeSelector asking for >= 80Gi would never bind.
      memory: { value: "81559Mi" }

# An InferenceCluster backed by a Nebius mk8s cluster. Modelplane provisions the
# full mk8s cluster and installs the inference stack; only the GPU node pool -
# referencing the InferenceClass above - is declared here. It authenticates to
# the cluster with the credentials of the Nebius ClusterProviderConfig named
# default, the same identity that provisions it.
#
# Delete with foreground cascading deletion for a clean teardown, so the
# inference stack uninstalls before the cluster's API server goes away:
#   kubectl delete inferencecluster nebius-eu-north --cascade=foreground
apiVersion: modelplane.ai/v1alpha1
kind: InferenceCluster
metadata:
  name: nebius-eu-north
  labels:
    modelplane.ai/region: eu-north
spec:
  cluster:
    source: Nebius
    nebius: {}
  nodePools:
  - name: gpu-h100
    className: h100-1x
    nodeCount: 1
    minNodeCount: 1
    maxNodeCount: 1

# Qwen3-8B served on a single NVIDIA H100 (Nebius) by vLLM, exposed on the
# Anthropic Messages API. vLLM's server registers /v1/messages (and
# /v1/messages/count_tokens) alongside its OpenAI routes with no extra flag, so
# this is an ordinary serve: the Anthropic surface comes for free, and
# Modelplane preserves the path below the service prefix to reach it.
#
# The tool-calling flags are what make it usable from Claude Code, not decoration:
#
#   --enable-auto-tool-choice with
#   --tool-call-parser=hermes        parse the model's tool calls so Claude Code's
#                                    tool use works (qwen3_xml is for Qwen3-Coder,
#                                    not this dense model). Qwen3's tool-use
#                                    template ships in the tokenizer, so no
#                                    --chat-template is needed.
#   --reasoning-parser=qwen3 with
#   --default-chat-template-kwargs   turns thinking off. Qwen3 thinks by default,
#                                    burying a one-line answer under a <think>
#                                    block and forbidding greedy decode.
#   --max-model-len=40960            Qwen3-8B's native context (past it needs YaRN).
#                                    Claude Code reserves 32000 output tokens, so a
#                                    smaller window 500s ("max_completion_tokens
#                                    cannot be greater than max_model_len", then
#                                    input+output overflow). The client must also
#                                    cap output with CLAUDE_CODE_MAX_OUTPUT_TOKENS
#                                    so input+output fit here. H100 has KV room.
#   --gpu-memory-utilization         headroom, not correctness.
#
# The v0.23.0 image is >0.17.1, so vLLM handles Claude Code's attribution header
# without breaking prefix caching. No --port or --host: Modelplane's routing
# expects the engine on its default :8000 with a /health probe, and passes args
# through verbatim.
apiVersion: modelplane.ai/v1alpha1
kind: ModelDeployment
metadata:
  name: qwen3-8b
  namespace: ml-team
spec:
  replicas: 1
  template:
    spec:
      # No clusterSelector: the single Nebius cluster is matched on device
      # capacity alone.
      engines:
      - name: qwen3-8b
        members:
        - role: Standalone
          nodeSelector:
            devices:
            - name: gpu
              count: 1
              selectors:
              - cel: |
                  device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("20Gi")) >= 0
          template:
            spec:
              containers:
              - name: engine
                image: vllm/vllm-openai:v0.23.0
                args:
                - "--model=Qwen/Qwen3-8B"
                - "--served-model-name=qwen"
                - "--max-model-len=40960"
                - "--gpu-memory-utilization=0.92"
                - "--reasoning-parser=qwen3"
                - "--default-chat-template-kwargs={\"enable_thinking\": false}"
                - "--enable-auto-tool-choice"
                - "--tool-call-parser=hermes"

# Exposes the qwen3-8b deployment as a single URL. The route matches the
# /<namespace>/<service>/ prefix and preserves the path below it, so the engine's
# Anthropic Messages API at .../v1/messages rides the same address as its
# OpenAI-compatible .../v1/chat/completions. Read the public address from
# status.address:
#   kubectl get ms qwen3-8b -n ml-team -o jsonpath='{.status.address}'
apiVersion: modelplane.ai/v1alpha1
kind: ModelService
metadata:
  name: qwen3-8b
  namespace: ml-team
spec:
  endpoints:
  - selector:
      matchLabels:
        modelplane.ai/deployment: qwen3-8b