Llama.cpp 可以在 CPU 和 GPU 環境中進行混合推理，這裡記錄一下在 Windows 10 平台上運行 Qwen-1.8B 的過程，顯卡是 1660Ti 。
準備模型#
conda create -n llamaConvert python=3.10 git -c conda-forge
conda activate llamaConvert
cd D:\llama
git clone --depth=1 https://github.com/ggerganov/llama.cpp.git
cd llama.cpp
python -m pip install -r requirements.txt
pip install tiktoken
python -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='Qwen/Qwen-1_8B-Chat', local_dir='D:\qwen', ignore_patterns=['*.h5', '*.ot', '*.msgpack', '*.safetensors'])"
cd D:\qwen
D:\aria2\aria2c.exe --all-proxy='http://127.0.0.1:7890' -o 'model-00001-of-00002.safetensors' "https://huggingface.co/Qwen/Qwen-1_8B-Chat/resolve/main/model-00001-of-00002.safetensors?download=true"
D:\aria2\aria2c.exe --all-proxy='http://127.0.0.1:7890' -o 'model-00002-of-00002.safetensors' "https://huggingface.co/Qwen/Qwen-1_8B-Chat/resolve/main/model-00002-of-00002.safetensors?download=true"
cd D:\llama\llama.cpp
python convert-hf-to-gguf.py D:\qwen
# Model successfully exported to 'D:\qwen\ggml-model-f16.gguf'
運行模型#
- 下載 llama-b1732-bin-win-cublas-cu12.2.0-x64.zip
- 提取文件到
D:\llama
conda create -n llamaCpp libcublas cuda-toolkit git -c nvidia -c conda-forge
conda activate llamaCpp
cd D:\llama | .\main.exe ## 檢查能否正確運行
cd D:\llama | .\quantize.exe --help ## 自己決定量化方式
.\quantize.exe D:\qwen\ggml-model-f16.gguf .\qwen-1_8-f16.gguf COPY
.\server.exe -m .\qwen-1_8-f16.gguf -c 4096 --n-gpu-layers 50 ## 調節 n-gpu-layers 平衡 CPU & GPU
- 訪問
http://127.0.0.1:8080選擇Completion進行測試