环境准备
1 |
|
1、创建docker环境
基础镜像：llm_management_20240910:py311sudo docker run --gpus all -it -d --network llm-management-network --name llm_management --ipc=host -P -p 8890-8899:8890-8899 -v /data/models:/models llm_management_20240910:py311 /bin/bash
进入docker容器：sudo docker exec -it b0efb809e606 /bin/bash
2、安装编译
编译步骤
1、官方文档
1 | # 1. 克隆代码 |
dependencies.sh文件如下
1 |
|
2、实际复现步骤
1、克隆代码：拉取https://github.com/kvcache-ai/Mooncake/并解压代码
2、安装依赖：运行./dependencies.sh
注意避坑：etcd-cpp-apiv3最新版本存在编译问题，请勿安装此版本。
注意避坑：python3.11会出现**error: invalid use of incomplete type ‘PyFrameObject’ \{aka ‘struct _frame’\}**[https://github.com/kvcache-ai/Mooncake/issues/15](https://github.com/kvcache-ai/Mooncake/issues/15) ，建议使用[**continuumio/miniconda3:22.11.1**](https://hub.docker.com/layers/continuumio/miniconda3/22.11.1/images/sha256-145567896379c33c89737aa1ab1389a71aead35b39f512118a13edcf9fe2a42c?context=explore)**镜像**
1 | git clone https://github.com/kvcache-ai/vllm.git |
3、创建 mooncake.json
配置文件准备：创建 mooncake.json：
1 | { |
- “prefill_url”: 预填充节点的 IP 地址和端口。
- URL 中的端口用于与 etcd 服务器通信以获取元数据。
- “decode_url”: 解码节点的 IP 地址和端口。
- URL 中的端口用于与 etcd 服务器通信以获取元数据。
- “metadata_server”: mooncake 传输引擎的 etcd 服务器。
- “protocol”: 数据传输协议(“rdma/tcp”)。
- “device_name”: 用于数据传输的设备，当 “protocol” 设置为 “rdma” 时必填。如果使用多个 NIC 设备，它们可以用逗号分隔，如 “erdma_0,erdma_1”。请注意它们之间没有空格。
使用 TCP 运行示例所需配置文件
1 | { |
4、测试步骤
1 | # 1. 启动etcd服务 |
启动etcd服务
启动预填充节点（生产者）
启动解码节点（消费者）
启动 proxy server节点（
python3 proxy_server.py）1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60import os
import aiohttp
from quart import Quart, make_response, request
AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=6 * 60 * 60)
app = Quart(__name__)
async def forward_request(url, data):
async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
headers = {
"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"
}
async with session.post(url=url, json=data,
headers=headers) as response:
if response.status == 200:
if True:
async for chunk_bytes in response.content.iter_chunked(
1024):
yield chunk_bytes
else:
content = await response.read()
yield content
async def handle_request():
try:
original_request_data = await request.get_json()
prefill_request = original_request_data.copy()
# change max_tokens = 1 to let it only do prefill
prefill_request['max_tokens'] = 1
# finish prefill
async for _ in forward_request('http://localhost:8100/v1/completions',
prefill_request):
continue
# return decode
generator = forward_request('http://localhost:8200/v1/completions', # Be sure to change the IP address for your machine
original_request_data)
response = await make_response(generator)
response.timeout = None
return response
except Exception as e:
import sys
import traceback
exc_info = sys.exc_info()
print("Error occurred in disagg prefill proxy server")
print(e)
print("".join(traceback.format_exception(*exc_info)))
if __name__ == '__main__':
app.run(host="0.0.0.0",port=8000)测试节点是否正常返回
1
2
3
4
5curl -s http://localhost:8000/v1/completions -H "Content-Type: application/json" -d '{
"model": "/models/Qwen2.5-0.5B-Instruct",
"prompt": "San Francisco is a",
"max_tokens": 100
}'
Mooncake与vllm适配测试
使用mooncake_20241205:py310镜像（已经打包编译好的环境）
使用mooncake_20241206:py310镜像（已经打包编译好的环境）
单机单卡Qwen2.5-0.5B-Instruct
1、基于mooncake_20241205:py310创建docker容器，内置编译好的环境sudo docker run –gpus all -it -d –network llm-management-network –name mooncake_node –ipc=host -P -p 8001-8004:8001-8004 -v /data/models:/models mooncake_20241205:py310 /bin/bash
sudo docker run –gpus all -it -d –network llm-management-network –name mooncake_node –ipc=host -P -p 8001-8021:8001-8021 -v /data/models:/models mooncake_20241206:py310 /bin/bash
2、配置文件准备：创建 mooncake.json：
使用 RDMA 运行示例所需配置文件
1 | { |
1 | { |
3、启动etcd服务
1 | etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://localhost:2379 |
4、启动预填充节点（在预填充侧运行（KVCache生产者））
在/mnt/Mooncake目录下（存在mooncake.json文件）执行命令
1 | VLLM_HOST_IP="0.0.0.0" VLLM_PORT="51000" MASTER_ADDR="0.0.0.0" MASTER_PORT="54324" MOONCAKE_CONFIG_PATH=./mooncake.json VLLM_DISTRIBUTED_KV_ROLE=producer VLLM_USE_MODELSCOPE=True CUDA_VISIBLE_DEVICES=0 python3 -m vllm.entrypoints.openai.api_server --model /models/Qwen2.5-0.5B-Instruct/ --port 8018 --max-model-len 100 --gpu-memory-utilization 0.08 |
5、启动解码节点（在解码侧运行（KVCache消费者））
在/mnt/Mooncake目录下（存在mooncake.json文件）执行命令
1 | VLLM_HOST_IP="0.0.0.0" VLLM_PORT="51000" MASTER_ADDR="0.0.0.0" MASTER_PORT="54324" MOONCAKE_CONFIG_PATH=./mooncake.json VLLM_DISTRIBUTED_KV_ROLE=consumer VLLM_USE_MODELSCOPE=True CUDA_VISIBLE_DEVICES=1 python3 -m vllm.entrypoints.openai.api_server --model /models/Qwen2.5-0.5B-Instruct/ --port 8019 --max-model-len 100 --gpu-memory-utilization 0.08 |
6、启动 proxy server节点（python3 proxy_server.py）
1 | import os |
测试节点是否正常返回
1
2
3
4
5curl -s http://localhost:8013/v1/completions -H "Content-Type: application/json" -d '{
"model": "/models/Qwen2.5-0.5B-Instruct/",
"prompt": "San Francisco is a",
"max_tokens": 100
}'结论
总结：单机单卡运行mooncake失败，其中prefill is ok, but decode stuckedhttps://github.com/kvcache-ai/Mooncake/issues/8多机单卡Qwen2.5-0.5B-Instruct
1、基于mooncake_20241206:py310创建docker容器，内置编译好的环境
sudo docker run –gpus all -it -d –network llm-management-network –name mooncake_node –ipc=host -P -p 12993-13013:12993-13013 -p 2369-2389:2369-2389 -p 50990-51010:50990-51010 -p 54314-54334:54314-54334 -p 8090-8110:8090-8110 -p 8190-8210:8190-8210 -v /data/models:/models mooncake_20241206:py310 /bin/bash
2、配置文件准备：创建 mooncake.json：
使用 RDMA 运行示例所需配置文件
1 | { |
1 | { |
上述服务端口为通信服务端口，与vllm本身服务无关。
3、启动etcd服务（在解码侧运行（etcd通信服务））
1 | etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://localhost:2379 |
4、启动预填充节点（在预填充侧运行（KVCache生产者））
在/mnt/Mooncake目录下（存在mooncake.json文件）执行命令
1 | VLLM_HOST_IP="10.39.214.111" VLLM_PORT="51000" MASTER_ADDR="10.39.214.111" MASTER_PORT="54324" MOONCAKE_CONFIG_PATH=./mooncake.json VLLM_DISTRIBUTED_KV_ROLE=producer VLLM_USE_MODELSCOPE=True CUDA_VISIBLE_DEVICES=0 python3 -m vllm.entrypoints.openai.api_server --model /models/Qwen2.5-0.5B-Instruct/ --port 8100 --max-model-len 100 --gpu-memory-utilization 0.08 |
5、启动解码节点（在解码侧运行（KVCache消费者））
在/mnt/Mooncake目录下（存在mooncake.json文件）执行命令
1 | VLLM_HOST_IP="10.39.214.111" VLLM_PORT="51000" MASTER_ADDR="10.39.214.111" MASTER_PORT="54324" MOONCAKE_CONFIG_PATH=./mooncake.json VLLM_DISTRIBUTED_KV_ROLE=consumer VLLM_USE_MODELSCOPE=True CUDA_VISIBLE_DEVICES=1 python3 -m vllm.entrypoints.openai.api_server --model /models/Qwen2.5-0.5B-Instruct/ --port 8200 --max-model-len 100 --gpu-memory-utilization 0.08 |
6、启动 proxy server节点（在预填充侧 python3 proxy_server.py）
1 | import os |
测试节点是否正常返回
1
2
3
4
5curl -s http://localhost:8090/v1/completions -H "Content-Type: application/json" -d '{
"model": "/models/Qwen2.5-0.5B-Instruct/",
"prompt": "San Francisco is a",
"max_tokens": 80
}'结论
服务正常部署调用性能测试：单机多卡Qwen2.5-72B-Instruct
首个 token 响应时长 (Time to first token, TTFT）
每个 token 输出时长 (Time per output token, TPOT)
跨 token 延迟 (Inter-token latency, ITL)
ITL是衡量服务器每次处理请求并返回一定数量token所需的时间，而TPOT是衡量服务器处理单个token所需的时间。
预填充服务：10.39.214.112:8100
解码服务：10.39.214.112:8200
分离式服务：10.39.214.112:8090
在A100（80G显存）机器上，以模型Qwen2.5-72B-Instruct为例。
vllm/benchmarks/disagg_benchmarks at mooncake-integration · kvcache-ai/vllm
Non-disaggregated
1、服务启动
1 | CUDA_VISIBLE_DEVICES=0,1,2,3 python3 \ |
2、执行python3 round_robin_proxy.py
round_robin_proxy.py
1 | import asyncio |
non_disagg_performance.sh
1 |
|
Non-disaggregated with **--enable-chunked-prefill** enabled
批量执行脚本non_disagg_performance.sh
1 |
|
1 | { |
2、或者配置RDMA所需服务
1 | { |
3、启动etcd服务（在解码侧运行（etcd通信服务））
1 | etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://localhost:2379 |
4、启动预填充节点（在预填充侧运行（KVCache生产者））
在/mnt/Mooncake目录下（存在mooncake.json文件）执行命令
1 | # Qwen2.5-72B-Instruct |
5、启动解码节点（在解码侧运行（KVCache消费者））
在/mnt/Mooncake目录下（存在mooncake.json文件）执行命令
1 | # Qwen2.5-72B-Instruct |
6、启动 disagg_prefill_proxy_server 节点（在预填充侧 python3 disagg_prefill_proxy_server.py）
1 | import os |
批量执行脚本disagg_tcp_performance.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
# Requirement: 8x A100 GPUs.
# Model: /models/Qwen2.5-72B-Instruct/
# Query: 1024 input tokens, 6 output tokens, QPS 2, 2000 requests
# Resource: 8x A100
# Approaches:
# 1. Chunked prefill: 1 vllm instance with tp=8
# 2. Chunked prefill: 2 vllm instance with tp=4, equivalent to 1 tp=4 instance with QPS 4
# 3. Disaggregated prefill: 1 prefilling instance and 1 decoding instance
# Prefilling instance: max_output_token=1
# Decoding instance: force the input tokens be the same across requests to bypass prefilling
set -ex
kill_gpu_processes() {
# kill all processes on GPU.
pkill -f pt_main_thread
pkill -f python3
ps -e | grep pt_main_thread | awk '{print $1}' | xargs kill -9
for port in 8090 8100 8200; do lsof -t -i:$port | xargs -r kill -9; done
sleep 1
}
wait_for_server() {
# wait for vllm server to start
# return 1 if vllm server crashes
local port=$1
timeout 1200 bash -c "
until curl -s localhost:${port}/v1/completions > /dev/null; do
sleep 1
done" && return 0 || return 1
}
launch_chunked_prefill() {
python3 disagg_prefill_proxy_server.py &
sleep 1
}
benchmark() {
results_folder="./results"
model="/models/Qwen2.5-72B-Instruct/"
dataset_name="sonnet"
dataset_path="../sonnet_4x.txt"
num_prompts=200
qps=$1
prefix_len=50
input_len=1024
output_len=$2
tag=$3
python3 ../benchmark_serving.py \
--backend vllm \
--model $model \
--dataset-name $dataset_name \
--dataset-path $dataset_path \
--sonnet-input-len $input_len \
--sonnet-output-len $output_len \
--sonnet-prefix-len $prefix_len \
--num-prompts $num_prompts \
--port 8090 \
--save-result \
--result-dir $results_folder \
--result-filename $tag-qps-$qps.json \
--request-rate $qps
sleep 2
}
main() {
(which wget && which curl) || (apt-get update && apt-get install -y wget curl)
(which jq) || (apt-get -y install jq)
(which socat) || (apt-get -y install socat)
pip install quart httpx matplotlib aiohttp
cd "$(dirname "$0")"
cd ..
# create sonnet-4x.txt so that we can sample 2048 tokens for input
echo "" > sonnet_4x.txt
for _ in {1..4}
do
cat sonnet.txt >> sonnet_4x.txt
done
cd disagg_benchmarks
# rm -rf results
mkdir results
default_output_len=6
export VLLM_LOGGING_LEVEL=DEBUG
export VLLM_HOST_IP=$(hostname -I | awk '{print $1}')
launch_chunked_prefill
for qps in 2 4 6 8; do
benchmark $qps $default_output_len disagg_tcp
done
kill_gpu_processes
}
main "$@"或者批量执行脚本disagg_rdma_performance.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
# Requirement: 8x A100 GPUs.
# Model: /models/Qwen2.5-72B-Instruct/
# Query: 1024 input tokens, 6 output tokens, QPS 2, 2000 requests
# Resource: 8x A100
# Approaches:
# 1. Chunked prefill: 1 vllm instance with tp=8
# 2. Chunked prefill: 2 vllm instance with tp=4, equivalent to 1 tp=4 instance with QPS 4
# 3. Disaggregated prefill: 1 prefilling instance and 1 decoding instance
# Prefilling instance: max_output_token=1
# Decoding instance: force the input tokens be the same across requests to bypass prefilling
set -ex
kill_gpu_processes() {
# kill all processes on GPU.
pkill -f pt_main_thread
pkill -f python3
ps -e | grep pt_main_thread | awk '{print $1}' | xargs kill -9
for port in 8090 8100 8200; do lsof -t -i:$port | xargs -r kill -9; done
sleep 1
}
wait_for_server() {
# wait for vllm server to start
# return 1 if vllm server crashes
local port=$1
timeout 1200 bash -c "
until curl -s localhost:${port}/v1/completions > /dev/null; do
sleep 1
done" && return 0 || return 1
}
launch_chunked_prefill() {
python3 disagg_prefill_proxy_server.py &
sleep 1
}
benchmark() {
results_folder="./results"
model="/models/Qwen2.5-72B-Instruct/"
dataset_name="sonnet"
dataset_path="../sonnet_4x.txt"
num_prompts=200
qps=$1
prefix_len=50
input_len=1024
output_len=$2
tag=$3
python3 ../benchmark_serving.py \
--backend vllm \
--model $model \
--dataset-name $dataset_name \
--dataset-path $dataset_path \
--sonnet-input-len $input_len \
--sonnet-output-len $output_len \
--sonnet-prefix-len $prefix_len \
--num-prompts $num_prompts \
--port 8090 \
--save-result \
--result-dir $results_folder \
--result-filename $tag-qps-$qps.json \
--request-rate $qps
sleep 2
}
main() {
(which wget && which curl) || (apt-get update && apt-get install -y wget curl)
(which jq) || (apt-get -y install jq)
(which socat) || (apt-get -y install socat)
pip install quart httpx matplotlib aiohttp
cd "$(dirname "$0")"
cd ..
# create sonnet-4x.txt so that we can sample 2048 tokens for input
echo "" > sonnet_4x.txt
for _ in {1..4}
do
cat sonnet.txt >> sonnet_4x.txt
done
cd disagg_benchmarks
# rm -rf results
mkdir results
default_output_len=6
export VLLM_LOGGING_LEVEL=DEBUG
export VLLM_HOST_IP=$(hostname -I | awk '{print $1}')
launch_chunked_prefill
for qps in 2 4 6 8; do
benchmark $qps $default_output_len disagg_rdma
done
kill_gpu_processes
}
main "$@"
MooncakeTransferEngine with TCP/RDMA backend（第二次—失败）
考虑不配置etcd、以及不直接启动预填充、解码服务。使用sh脚本启动服务（disagg_default）。
1 |
|
