GLM-4-Voice，智谱开源版“Her”来了！

git clone https://github.com/THUDM/GLM-4-Voice.git
 
pip install matcha-tts torchaudio hyperpyyaml
cd GLM-4-Voice
# 如果出现环境问题，可以运行以下命令
pip install -r requirements.txt

import os
import uuid
from typing import List, Optional, Tuple
import torch
import torchaudio
from flow_inference import AudioDecoder
from modelscope import snapshot_download
from speech_tokenizer.modeling_whisper import WhisperVQEncoder
from speech_tokenizer.utils import extract_speech_token
from transformers import AutoModel, AutoTokenizer, GenerationConfig, WhisperFeatureExtractor
class GLM4Voice:
    def _prepare_model(self):
        model_path = snapshot_download('ZhipuAI/glm-4-voice-9b')
        decoder_path = snapshot_download('ZhipuAI/glm-4-voice-decoder')
        tokenizer_path = snapshot_download('ZhipuAI/glm-4-voice-tokenizer')
        flow_config = os.path.join(decoder_path, 'config.yaml')
        flow_checkpoint = os.path.join(decoder_path, 'flow.pt')
        hift_checkpoint = os.path.join(decoder_path, 'hift.pt')
        # GLM
        self.model = AutoModel.from_pretrained(model_path, trust_remote_code=True, device_map=self.device).eval()
        self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True, use_fast=False)
        # Flow & Hift
        self.audio_decoder = AudioDecoder(
            config_path=flow_config, flow_ckpt_path=flow_checkpoint, hift_ckpt_path=hift_checkpoint, device=self.device)
        # Speech tokenizer
        self.whisper_model = WhisperVQEncoder.from_pretrained(tokenizer_path).eval().to(self.device)
        self.feature_extractor = WhisperFeatureExtractor.from_pretrained(tokenizer_path)
    def clear(self):
        self.previous_tokens = ''
    def __init__(self, generation_config=None):
        if generation_config is None:
            generation_config = GenerationConfig(top_p=0.8, temperature=0.2, max_new_tokens=2000, do_sample=True)
        self.generation_config = generation_config
        self.device = 'cuda:0'
        self._prepare_model()
        self.audio_offset = self.tokenizer.convert_tokens_to_ids('<|audio_0|>')
        self.end_token_id = self.tokenizer.convert_tokens_to_ids('<|user|>')
        self.clear()
    def infer(self, audio_path: Optional[str] = None, text: Optional[str] = None) -> Tuple[str, str]:
        if audio_path is not None:
            audio_tokens = extract_speech_token(self.whisper_model, self.feature_extractor, [audio_path])[0]
            audio_tokens = ''.join([f'<|audio_{x}|>' for x in audio_tokens])
            audio_tokens = '<|begin_of_audio|>' + audio_tokens + '<|end_of_audio|>'
            user_input = audio_tokens
            system_prompt = 'User will provide you with a speech instruction. Do it step by step. First, think about the instruction and respond in a interleaved manner, with 13 text token followed by 26 audio tokens. '
        else:
            user_input = text
            system_prompt = 'User will provide you with a text instruction. Do it step by step. First, think about the instruction and respond in a interleaved manner, with 13 text token followed by 26 audio tokens.'
        text = self.previous_tokens
        text = text.strip()
        if '<|system|>' not in text:
            text += f'<|system|>\n{system_prompt}'
        text += f'<|user|>\n{user_input}<|assistant|>streaming_transcription\n'
        inputs = self.tokenizer([text], return_tensors='pt').to(self.device)
        generate_ids = self.model.generate(**inputs, generation_config=self.generation_config)[0]
        generate_ids = generate_ids[inputs['input_ids'].shape[1]:]
        self.previous_tokens += text + self.tokenizer.decode(generate_ids, spaces_between_special_tokens=False)
        return self._parse_generate_ids(generate_ids)
    def _parse_generate_ids(self, generate_ids: List[int]) -> Tuple[str, str]:
        text_tokens, audio_tokens = [], []
        this_uuid = str(uuid.uuid4())
        for token_id in generate_ids.tolist():
            if token_id >= self.audio_offset:
                audio_tokens.append(token_id - self.audio_offset)
            elif token_id != self.end_token_id:
                text_tokens.append(token_id)
        audio_tokens_pt = torch.tensor(audio_tokens, device=self.device)[None]
        tts_speech, _ = self.audio_decoder.token2wav(audio_tokens_pt, uuid=this_uuid, finalize=True)
        audio_path = f'{this_uuid}.wav'
        with open(audio_path, 'wb') as f:
            torchaudio.save(f, tts_speech.cpu(), 22050, format='wav')
        response = self.tokenizer.decode(text_tokens)
        return response, audio_path
if __name__ == '__main__':
    generation_config = GenerationConfig(top_p=0.8, temperature=0.2, max_new_tokens=2000, do_sample=True)
    glm_voice = GLM4Voice(generation_config=generation_config)
    audio_path = 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/weather.wav'
    response, output_path = glm_voice.infer(audio_path=audio_path)
    print(f'response: {response}\noutput_path: {output_path}')
    text = '请用英文回答'
    response, output_path = glm_voice.infer(text=text)
    print(f'response: {response}\noutput_path: {output_path}')
    glm_voice.clear()  # 清空历史
    text = '请用英文回答'
    response, output_path = glm_voice.infer(text=text)
    print(f'response: {response}\noutput_path: {output_path}')
"""
response: 是啊，阳光明媚的，真是个出门走走的好日子！你今天有什么计划吗？
output_path: 7f146cb5-4c1f-4c2c-85d0-0a8c985c90c0.wav
response: Sure! Today's weather is really nice, isn't it? It's a great day to go out and enjoy some fresh air. Do you have any plans for today?
output_path: 9326df35-aeec-4292-856b-5c0b1688e3f8.wav
response: Sure, I'll answer in English. What would you like to know?
output_path: e6e7c94b-7532-475f-bea7-e41566a954b6.wav
"""

git clone --recurse-submodules https://github.com/THUDM/GLM-4-Voice
cd GLM-4-Voice

pip install -r requirements.txt

# 使用ModelScope CLI下载
modelscope download --model=ZhipuAI/glm-4-voice-9b --local_dir ./glm-4-voice-9b
# 建议其他模型也单独下载
modelscope download --model=ZhipuAI/glm-4-voice-decoder --local_dir ./glm-4-voice-decoder
modelscope download --model=ZhipuAI/glm-4-voice-tokenizer --local_dir ./glm-4-voice-tokenizer

python model_server.py --model-path glm-4-voice-9b

python web_demo.py --tokenizer-path glm-4-voice-tokenizer --model-path glm-4-voice-9b