vllm+ray+环境变量

Michaelwubo

252人浏览 · 2026-04-03 11:26:53

Michaelwubo · 2026-04-03 11:26:53 发布

1. pytorch
https://pytorch.org/get-started/previous-versions/

2. ray 算力集群
https://docs.ray.io/en/latest/ray-overview/installation.html

3. vllm 模型推理
https://vllm-zh.llamafactory.cn/serving/deploying_with_docker.html

4.
cuda tools： 这个是基础
nccl：这个是分部署底层数据传输库
tensorrt: 底层推理引擎

apt install -y libnccl2 libnccl-dev --allow-unauthenticated
apt install -y tensorrt --allow-unauthenticated



ldconfig -p | grep nvinfer
ldconfig -p | grep nccl


export NCCL_SOCKET_IFNAME=eth0
export GLOO_SOCKET_IFNAME=eth0
export NCCL_IB_DISABLE=1
export NCCL_P2P_DISABLE=1
export NCCL_IGNORE_DISABLED_NICS=1
export NCCL_DEBUG=INFO
export VLLM_HOST_IP=$(hostname -I | awk '{print $1}')



docker run -it --name vllm --ipc=host --network host --entrypoint /bin/bash --gpus all -v /opt//work/wubo:/opt/jettech  harbor.jettech.com/jettechtools/jettech-inference-tritonserver:25.11-vllm-python-py3.aarch64
docker run -it --name vllm --ipc=host --pid=host --network host --entrypoint /bin/bash --gpus all -v /opt//work/wubo:/opt/jettech  harbor.jettech.com/jettechtools/jettech-inference-tritonserver:25.11-vllm-python-py3.aarch64


vllm  serve --host=0.0.0.0 --port=8080 /opt/jettech/models/Qwen2.5-VL-32B-Instruct --served-model-name="Qwen2.5-VL-32B-Instruct"  --trust-remote-code  --enable-prefix-caching  --tensor-parallel-size=1 --max-num-seqs=4 --max-model-len=8192  --block-size=128 --gpu-memory-utilization=0.9 --limit-mm-per-prompt='{"image": '10'}'


cluster
ray start --head --port=6379 --dashboard-host=0.0.0.0 --node-ip-address=192.168.0.191
ray start --head --dashboard-host=0.0.0.0   --node-ip-address=192.168.0.191 --port=6379 --num-gpus=0
add node
ray start --address='192.168.0.73:6379'


vllm  serve --host=0.0.0.0 --port=8080 /opt/jettech/models/Qwen2.5-VL-32B-Instruct --served-model-name="Qwen2.5-VL-32B-Instruct"   --tensor-parallel-size=1 --pipeline-parallel-size=3 --distributed-executor-backend=ray

vllm  serve --host=0.0.0.0 --port=8080 /opt/jettech/models/Qwen2.5-VL-32B-Instruct --served-model-name="Qwen2.5-VL-32B-Instruct"  --trust-remote-code   --tensor-parallel-size=2  --distributed-executor-backend=ray --max-model-len 8192 --gpu-memory-utilization 0.95 



1. docker run -it --name vllm --ipc=host --network host --entrypoint /bin/bash --gpus all -v /opt//work/wubo:/opt/jettech  harbor.jettech.com/jettechtools/jettech-inference-tritonserver:25.11-vllm-python-py3.aarch64
2. ray start --head --port=6379 --dashboard-host=0.0.0.0 --node-ip-address=192.168.0.191
3. vllm  serve --host=0.0.0.0 --port=8080 --model /opt/work/wubo/models/Qwen2.5-VL-32B-Instruct --served-model-name="Qwen2.5-VL-32B-Instruct"   --tensor-parallel-size=1 --pipeline-parallel-size=1 --distributed-executor-backend=ray --trust-remote-code  --enable-prefix-caching  --max-model-len=8192  --block-size=128 --gpu-memory-utilization=0.9 --limit-mm-per-prompt='{"image": '10'}'


python3 -m vllm.entrypoints.openai.api_server --model /opt/jettech/models/Qwen2.5-VL-32B-Instruct --served-model-name Qwen2.5-VL-32B-Instruct --host 0.0.0.0 --port 8080 --block-size 16  --pipeline-parallel-size 2 --tensor-parallel-size=1 --trust-remote-code --enforce-eager




export VLLM_HOST_IP=192.168.0.191
ray start --head   --node-ip-address=192.168.0.191   --port=6379   --include-dashboard=True   --dashboard-host=0.0.0.0   --dashboard-port=8265   --num-gpus=1   --resources='{"node:192.168.0.191": 1}'

export VLLM_HOST_IP=192.168.0.124
ray start --address="192.168.0.191:6379"   --node-ip-address=192.168.0.124  --num-gpus=1   --resources='{"node:192.168.0.124": 1}'

(python3.12.3) root@spark-4cd7:~# cat /etc/os-release 
PRETTY_NAME="Ubuntu 24.04.4 LTS"
NAME="Ubuntu"
VERSION_ID="24.04"
VERSION="24.04.4 LTS (Noble Numbat)"
VERSION_CODENAME=noble
ID=ubuntu
ID_LIKE=debian
HOME_URL="https://www.ubuntu.com/"
SUPPORT_URL="https://help.ubuntu.com/"
BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
UBUNTU_CODENAME=noble
LOGO=ubuntu-logo

export PATH=/usr/local/cuda/bin:$PATH
export LD_LIBRARY_PATH="/usr/local/cuda/lib64:$LD_LIBRARY_PATH"


conda  create --name python3.12.3 python=3.12.3
pip3 install vllm==0.18.1 -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host pypi.tuna.tsinghua.edu.cn


(python3.12.3) root@spark-53df:~# dpkg -l | grep tensorrt
ii  tensorrt                                         10.16.0.72-1+cuda13.2                    arm64        Meta package for TensorRT


(python3.12.3) root@spark-53df:~# dpkg -l | grep libnccl2 
ii  libnccl2                                         2.29.7-1+cuda13.2                        arm64        NVIDIA Collective Communication Library (NCCL) Runtime
(python3.12.3) root@spark-53df:~# dpkg -l | grep libnccl-dev
ii  libnccl-dev                                      2.29.7-1+cuda13.2                        arm64        NVIDIA Collective Communication Library (NCCL) Development Files



apt install -y libnccl2 libnccl-dev --allow-unauthenticated
apt install -y tensorrt --allow-unauthenticated




ldconfig -p | grep nvinfer
ldconfig -p | grep nccl


export NCCL_SOCKET_IFNAME=enP7s7
export GLOO_SOCKET_IFNAME=enP7s7
export NCCL_IB_DISABLE=1
export NCCL_P2P_DISABLE=1
export NCCL_IGNORE_DISABLED_NICS=1
export NCCL_DEBUG=INFO
export VLLM_HOST_IP=$(hostname -I | awk '{print $1}')



ray start --head --port=6379 --dashboard-host=0.0.0.0 --node-ip-address=192.168.0.191
ray start --address='192.168.0.191:6379' --node-ip-address=192.168.0.191


计算注意力头数
python -c "from transformers import AutoConfig; c=AutoConfig.from_pretrained('/opt/work/wubo/models/Qwen2.5-VL-32B-Instruct'); print('layers:', c.num_hidden_layers, 'heads:', c.num_attention_heads, 'hidden:', c.hidden_size)"

TP:张量并发计算   单层矩阵进行分开子矩阵向量到各自GPU
PP:流水线并发计算 对多层进行分开到不同GPU 是以层的维度分开 

vllm  serve --host=0.0.0.0 --port=8080 --model /opt/work/wubo/models/Qwen2.5-VL-32B-Instruct --served-model-name="Qwen2.5-VL-32B-Instruct"   --tensor-parallel-size=1 --pipeline-parallel-size=4 --distributed-executor-backend=ray  --trust-remote-code --max-model-len 8192 --gpu-memory-utilization 0.85  --limit-mm-per-prompt='{"image": '10'}'

vllm  serve --host=0.0.0.0 --port=8080 --model /opt/work/wubo/models/Qwen2.5-VL-72B-Instruct --served-model-name="Qwen2.5-VL-72B-Instruct"   --tensor-parallel-size=1 --pipeline-parallel-size=4 --distributed-executor-backend=ray  --trust-remote-code --max-model-len 8192 --gpu-memory-utilization 0.85  --limit-mm-per-prompt='{"image": '10'}'

curl -X POST  -H "Content-Type: application/json"  http://192.168.0.191:8080/v1/chat/completions -d '{"model": "Qwen2.5-VL-32B-Instruct","messages": [{"role": "user","content": "你好吗？什么 是自动化测试？"}],"stream":false}'

cd /
tar -cvpf ubuntu-system.tar --exclude=/proc --exclude=/sys --exclude=/dev --exclude=/tmp --exclude=/run --exclude=/mnt --exclude=/media --exclude=/lost+found --exclude=/var/lib/docker --exclude=/wubo --exclude=/opt/work  /
(base) root@spark-53df:/wubo# cat ubuntu-system.tar | sudo docker import - my-ubuntu-system:v1



docker run -it -v /opt/work/wubo:/opt/work/wubo  --net=host --gpus all --ipc=host   -d harbor.jettech.com/jettechtools/jettech-inference:vllm-gdx-cuda12-nccl-tersort-aarch64-py3.12.3