metahuman-stream是基于ernerf模型的流式数字人，实现音视频同步对话。
metahuman-stream
xtts-streaming-server
srs
部署 srs
# rtmpserver
docker run -it -d \
-p 1935:1935 -p 1985:1985 -p 8080:8080 -p 8000:8000/udp -p 10080:10080/udp \
--name srs \
registry.cn-hangzhou.aliyuncs.com/ossrs/srs:5
部署 xtts-streaming-server
下载XTTS-v2模型到宿主机
docker run --gpus=all -d -p 9000:80 \
-e COQUI_TOS_AGREED=1 \
-v /opt/data/model/XTTS-v2:/app/tts_models \
--name him-xtts-streaming-server \
ghcr.io/coqui-ai/xtts-streaming-server:latest
部署 metahuman-stream
参考项目的Dockerfile
启动容器
sudo docker run --gpus all -it -d \
--network=host \
--name nerfstream \
nvcr.io/nvidia/cuda:11.6.1-cudnn8-devel-ubuntu20.04
sudo docker exec -ti nerfstream bash
安装工具和anaconda
apt-get update -yq --fix-missing \
&& DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
pkg-config \
wget \
cmake \
curl \
git \
vim
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
sh Miniconda3-latest-Linux-x86_64.sh -b -u -p ~/miniconda3
~/miniconda3/bin/conda init
source ~/.bashrc
创建conda环境&安装依赖
conda create -n nerfstream python=3.10
conda activate nerfstream
pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/
conda install pytorch==1.12.1 torchvision==0.13.1 cudatoolkit=11.3 -c pytorch
pip install "git+https://github.com/facebookresearch/pytorch3d.git"
pip install tensorflow-gpu==2.8.0
pip uninstall protobuf
pip install protobuf==3.20.1
conda install ffmpeg
apt-get install portaudio19-dev python-all-dev python3-all-dev
conda install pyaudio
克隆源码&安装项目依赖
cd /
git clone https://github.com/lipku/metahuman-stream.git
cd metahuman-stream
pip install -r requirements.txt
cd /
git clone https://github.com/lipku/python_rtmpstream.git
cd python_rtmpstream
git submodule update --init
pip install wheel
cd python
# 修改conda目录
vim CMakeLists.txt
pip install .
配置LLM
以Qwen为例，修改app.py
def llm_response(message):
from llm.LLM import LLM
llm = LLM().init_model('Qwen', model_path= 'Qwen/Qwen-1_8B-Chat')
response = llm.chat(message)
print(response)
return response
运行
export HF_ENDPOINT=https://hf-mirror.com
python app.py # 默认
python app.py --tts xtts --ref_file data/ref.wav #指定克隆的语音
效果
