NVIDIA Driver + vLLM Complete Setup Guide

# Docker
sudo apt install docker.io
sudo usermod -aG docker $USER

# NVIDIA Container Toolkit (enables --gpus flag)
# https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | \
  sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
  sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
  sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
sudo apt update
sudo apt install nvidia-container-toolkit
sudo nvidia-ctk runtime configure --runtime=docker
sudo systemctl restart docker

ubuntu-drivers devices

sudo apt update
sudo apt install nvidia-driver-590-open

sudo ubuntu-drivers install

# Check installed drivers
dpkg -l | grep nvidia-driver

# Remove old versions (adjust package names as needed)
sudo apt remove --purge nvidia-driver-580-open nvidia-driver-575-open
sudo apt autoremove

sudo reboot

nvidia-smi

docker rm -f vllm-qwen 2>/dev/null

docker pull vllm/vllm-openai:latest

docker run -d --gpus all \
  -e LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:/usr/local/cuda/lib64 \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  -p 8000:8000 \
  --name vllm-qwen \
  vllm/vllm-openai:latest \
  --model Qwen/Qwen2.5-7B-Instruct \
  --max-model-len 4096 \
  --enforce-eager

docker logs -f vllm-qwen

curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "Qwen/Qwen2.5-7B-Instruct",
    "messages": [{"role": "user", "content": "Hello"}],
    "max_tokens": 32
  }'

{
  "model": "Qwen/Qwen2.5-7B-Instruct",
  "choices": [{
    "message": {
      "role": "assistant",
      "content": "Hello! How can I assist you today?"
    },
    "finish_reason": "stop"
  }]
}

docker exec vllm-qwen nvidia-smi

curl -s http://localhost:8000/v1/models | python3 -m json.tool

# Follow logs
docker logs -f vllm-qwen

# Stop server
docker stop vllm-qwen

# Restart stopped server
docker start vllm-qwen

# Remove container
docker stop vllm-qwen && docker rm vllm-qwen

# 1. Verify driver loaded
nvidia-smi

# 2. Remove stale container and start fresh
docker rm -f vllm-qwen 2>/dev/null
docker run -d --gpus all \
  -e LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:/usr/local/cuda/lib64 \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  -p 8000:8000 \
  --name vllm-qwen \
  vllm/vllm-openai:latest \
  --model Qwen/Qwen2.5-7B-Instruct \
  --max-model-len 4096 \
  --enforce-eager

# 3. Wait for startup (~1-2 min)
docker logs -f vllm-qwen
# Look for: "Application startup complete."
# Ctrl+C to exit logs

# 4. Test
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{"model":"Qwen/Qwen2.5-7B-Instruct","messages":[{"role":"user","content":"Hello"}],"max_tokens":32}'

# Check if kernel module loaded
lsmod | grep nvidia

# Check for errors
dmesg | grep -i nvidia

# Reinstall driver if needed
sudo apt install --reinstall nvidia-driver-590-open
sudo reboot

# Find what's using the port
sudo lsof -i :8000

# Or use a different port
docker run -d --gpus all ... -p 8001:8000 ...
# Then access via http://localhost:8001

curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "Qwen/Qwen2.5-7B-Instruct",
    "messages": [
      {"role": "system", "content": "You are a helpful assistant."},
      {"role": "user", "content": "Explain Docker in one sentence."}
    ],
    "max_tokens": 128,
    "temperature": 0.7
  }'

curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "Qwen/Qwen2.5-7B-Instruct",
    "messages": [{"role": "user", "content": "Tell me a joke"}],
    "max_tokens": 128,
    "stream": true
  }'

from openai import OpenAI

client = OpenAI(base_url="http://localhost:8000/v1", api_key="not-needed")

response = client.chat.completions.create(
    model="Qwen/Qwen2.5-7B-Instruct",
    messages=[{"role": "user", "content": "Hello"}],
    max_tokens=128,
)
print(response.choices[0].message.content)