LLM Inference Optimization Part 4 — Production Serving

from vllm import LLM, SamplingParams

llm = LLM(
    model="meta-llama/Llama-3.1-8B-Instruct",
    dtype="float16",

    # === Memory Management ===
    gpu_memory_utilization=0.90,   # Use 90% of GPU memory
    max_model_len=32768,            # Maximum context length

    # === KV Cache Optimization ===
    kv_cache_dtype="auto",          # "auto", "fp8_e5m2", "fp8_e4m3"
    # kv_cache_dtype="fp8_e5m2",    # FP8 KV Cache → 2x memory savings

    # === Quantization ===
    # quantization="awq",           # Model weight quantization

    # === Parallelism ===
    tensor_parallel_size=1,         # Number of GPUs
)

params = SamplingParams(
    temperature=0.7,
    top_p=0.9,
    max_tokens=1024,
    stop=["<|eot_id|>"],
)

output = llm.generate("Explain quantum computing.", params)
print(output[0].outputs[0].text)

# Start vLLM server
python -m vllm.entrypoints.openai.api_server \
    --model meta-llama/Llama-3.1-8B-Instruct \
    --dtype float16 \
    --gpu-memory-utilization 0.9 \
    --max-model-len 32768 \
    --kv-cache-dtype fp8_e5m2 \
    --port 8000