File size: 12,459 Bytes
f1a0148
 
 
 
782d74b
 
f1a0148
 
 
 
 
dab4429
 
f1a0148
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1edfb59
 
 
 
 
 
 
 
 
 
 
 
f1a0148
 
 
 
1edfb59
 
 
 
 
 
 
 
19ae156
 
 
 
1edfb59
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
87f7c84
4172058
 
 
 
9e032ec
 
 
 
f1a0148
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1edfb59
 
 
 
 
 
 
 
 
 
 
 
d25b36f
 
dab4429
1edfb59
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
dab4429
1edfb59
 
 
19ae156
782d74b
 
57c2abe
782d74b
 
57c2abe
782d74b
 
 
 
 
 
 
 
 
 
19ae156
 
782d74b
 
 
 
 
 
 
 
 
 
 
 
dab4429
4172058
 
 
782d74b
4172058
 
 
 
 
 
 
 
 
 
 
19ae156
 
 
9e032ec
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1edfb59
f1a0148
 
 
 
 
 
 
 
 
1edfb59
 
 
 
19ae156
 
4172058
 
9e032ec
 
f1a0148
 
 
 
1edfb59
 
 
 
 
 
4172058
1edfb59
 
 
 
 
 
 
 
 
 
 
 
 
f1a0148
 
 
1edfb59
f1a0148
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
# TODO: V2 of TTS Router
# Currently just use current TTS router.
import os
import json
import sys

from dotenv import load_dotenv
import fal_client
import requests
import time
import io

from gradio_client import handle_file
from pyht import Client as PyhtClient
from pyht.client import TTSOptions
import base64
import tempfile
import random

load_dotenv()

ZEROGPU_TOKENS = os.getenv("ZEROGPU_TOKENS", "").split(",")


def get_zerogpu_token():
    return random.choice(ZEROGPU_TOKENS)


model_mapping = {
    # "eleven-multilingual-v2": {
    #     "provider": "elevenlabs",
    #     "model": "eleven_multilingual_v2",
    # },
    # "eleven-turbo-v2.5": {
    #     "provider": "elevenlabs",
    #     "model": "eleven_turbo_v2_5",
    # },
    # "eleven-flash-v2.5": {
    #     "provider": "elevenlabs",
    #     "model": "eleven_flash_v2_5",
    # },
    "spark-tts": {
        "provider": "spark",
        "model": "spark-tts",
    },
    # "playht-2.0": {
    #     "provider": "playht",
    #     "model": "PlayHT2.0",
    # },
    # "styletts2": {
    #     "provider": "styletts",
    #     "model": "styletts2",
    # },
    "cosyvoice-2.0": {
        "provider": "cosyvoice",
        "model": "cosyvoice_2_0",
    },
    # "papla-p1": {
    #     "provider": "papla",
    #     "model": "papla_p1",
    # },
    # "hume-octave": {
    #     "provider": "hume",
    #     "model": "octave",
    # },
    # "minimax-02-hd": {
    #     "provider": "minimax",
    #     "model": "speech-02-hd",
    # },
    # "minimax-02-turbo": {
    #     "provider": "minimax",
    #     "model": "speech-02-turbo",
    # },
    # "lanternfish-1": {
    #     "provider": "lanternfish",
    #     "model": "lanternfish-1",
    # },
    "index-tts": {
        "provider": "bilibili",
        "model": "index-tts",
    },
    "maskgct": {
        "provider": "amphion",
        "model": "maskgct",
    },
    "gpt-sovits-v2": {
        "provider": "gpt-sovits",
        "model": "gpt-sovits-v2",
    },
}
url = "https://tts-agi-tts-router-v2.hf.space/tts"
headers = {
    "accept": "application/json",
    "Content-Type": "application/json",
    "Authorization": f'Bearer {os.getenv("HF_TOKEN")}',
}
data = {"text": "string", "provider": "string", "model": "string"}


def predict_csm(script):
    result = fal_client.subscribe(
        "fal-ai/csm-1b",
        arguments={
            # "scene": [{
            #     "text": "Hey how are you doing.",
            #     "speaker_id": 0
            # }, {
            #     "text": "Pretty good, pretty good.",
            #     "speaker_id": 1
            # }, {
            #     "text": "I'm great, so happy to be speaking to you.",
            #     "speaker_id": 0
            # }]
            "scene": script
        },
        with_logs=True,
    )
    return requests.get(result["audio"]["url"]).content


def predict_playdialog(script):
    # Initialize the PyHT client
    pyht_client = PyhtClient(
        user_id=os.getenv("PLAY_USERID"),
        api_key=os.getenv("PLAY_SECRETKEY"),
    )

    # Define the voices
    voice_1 = "s3://voice-cloning-zero-shot/baf1ef41-36b6-428c-9bdf-50ba54682bd8/original/manifest.json"
    voice_2 = "s3://voice-cloning-zero-shot/e040bd1b-f190-4bdb-83f0-75ef85b18f84/original/manifest.json"

    # Convert script format from CSM to PlayDialog format
    if isinstance(script, list):
        # Process script in CSM format (list of dictionaries)
        text = ""
        for turn in script:
            speaker_id = turn.get("speaker_id", 0)
            prefix = "Host 1:" if speaker_id == 0 else "Host 2:"
            text += f"{prefix} {turn['text']}\n"
    else:
        # If it's already a string, use as is
        text = script

    # Set up TTSOptions
    options = TTSOptions(
        voice=voice_1, voice_2=voice_2, turn_prefix="Host 1:", turn_prefix_2="Host 2:"
    )

    # Generate audio using PlayDialog
    audio_chunks = []
    for chunk in pyht_client.tts(text, options, voice_engine="PlayDialog"):
        audio_chunks.append(chunk)

    # Combine all chunks into a single audio file
    return b"".join(audio_chunks)


def predict_dia(script):
    # Convert script to the required format for Dia
    if isinstance(script, list):
        # Convert from list of dictionaries to formatted string
        formatted_text = ""
        for turn in script:
            speaker_id = turn.get("speaker_id", 0)
            speaker_tag = "[S1]" if speaker_id == 0 else "[S2]"
            text = turn.get("text", "").strip().replace("[S1]", "").replace("[S2]", "")
            formatted_text += f"{speaker_tag} {text} "
        text = formatted_text.strip()
    else:
        # If it's already a string, use as is
        text = script
    print(text)
    # Make a POST request to initiate the dialogue generation
    headers = {
        # "Content-Type": "application/json",
        "Authorization": f"Bearer {get_zerogpu_token()}"
    }

    response = requests.post(
        "https://mrfakename-dia-1-6b.hf.space/gradio_api/call/generate_dialogue",
        headers=headers,
        json={"data": [text]},
    )

    # Extract the event ID from the response
    event_id = response.json()["event_id"]

    # Make a streaming request to get the generated dialogue
    stream_url = f"https://mrfakename-dia-1-6b.hf.space/gradio_api/call/generate_dialogue/{event_id}"

    # Use a streaming request to get the audio data
    with requests.get(stream_url, headers=headers, stream=True) as stream_response:
        # Process the streaming response
        for line in stream_response.iter_lines():
            if line:
                if line.startswith(b"data: ") and not line.startswith(b"data: null"):
                    audio_data = line[6:]
                    return requests.get(json.loads(audio_data)[0]["url"]).content


def predict_index_tts(text, reference_audio_path=None):
    from gradio_client import Client, handle_file
    client = Client("IndexTeam/IndexTTS")
    if reference_audio_path:
        prompt = handle_file(reference_audio_path)
    else:
        raise ValueError("index-tts 需要 reference_audio_path")
    result = client.predict(
        prompt=prompt,
        text=text,
        api_name="/gen_single"
    )
    if type(result) != str:
        result = result.get("value")
    print("index-tts result:", result)
    return result


def predict_spark_tts(text, reference_audio_path=None):
    from gradio_client import Client, handle_file
    client = Client("amortalize/Spark-TTS-Zero")
    prompt_wav = None
    if reference_audio_path:
        prompt_wav = handle_file(reference_audio_path)
    result = client.predict(
        text=text,
        prompt_text=text,
        prompt_wav_upload=prompt_wav,
        prompt_wav_record=prompt_wav,
        api_name="/voice_clone"
    )
    print("spark-tts result:", result)
    return result


def predict_cosyvoice_tts(text, reference_audio_path=None):
    import tempfile
    import soundfile as sf
    from huggingface_hub import snapshot_download
    model_dir = os.path.join(os.path.dirname(__file__), "CosyVoice2-0.5B", "pretrained_models", "CosyVoice2-0.5B")
    if not os.path.exists(model_dir) or not os.listdir(model_dir):
        snapshot_download('FunAudioLLM/CosyVoice2-0.5B', local_dir=model_dir)
    sys.path.append(os.path.join(os.path.dirname(__file__), "CosyVoice2-0.5B"))
    from cosyvoice.cli.cosyvoice import CosyVoice2
    from cosyvoice.utils.file_utils import load_wav

    # 全局模型初始化
    global _cosyvoice_model
    if '_cosyvoice_model' not in globals() or _cosyvoice_model is None:
        _cosyvoice_model = CosyVoice2(model_dir)
    model = _cosyvoice_model

    if not reference_audio_path:
        raise ValueError("cosyvoice-2.0 需要 reference_audio_path")
    # 读取参考音频
    prompt_speech_16k = load_wav(reference_audio_path, 16000)
    # 参考文本可选,这里不做ASR,直接传空字符串
    prompt_text = ""
    # 推理
    result = None
    for i in model.inference_zero_shot(text, prompt_text, prompt_speech_16k):
        result = i['tts_speech'].numpy().flatten()
    # 保存为临时wav
    temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".wav")
    sf.write(temp_file.name, result, 24000)
    return temp_file.name


def predict_maskgct(text, reference_audio_path=None):
    from gradio_client import Client, handle_file
    client = Client("cocktailpeanut/maskgct")
    if not reference_audio_path:
        raise ValueError("maskgct 需要 reference_audio_path")
    prompt_wav = handle_file(reference_audio_path)
    result = client.predict(
        prompt_wav=prompt_wav,
        target_text=text,
        target_len=-1,
        n_timesteps=25,
        api_name="/predict"
    )
    print("maskgct result:", result)
    return result


def predict_gpt_sovits_v2(text, reference_audio_path=None):
    from gradio_client import Client, file
    client = Client("lj1995/GPT-SoVITS-v2")
    if not reference_audio_path:
        raise ValueError("GPT-SoVITS-v2 需要 reference_audio_path")
    result = client.predict(
        ref_wav_path=file(reference_audio_path),
        prompt_text="",
        prompt_language="English",
        text=text,
        text_language="English",
        how_to_cut="Slice once every 4 sentences",
        top_k=15,
        top_p=1,
        temperature=1,
        ref_free=False,
        speed=1,
        if_freeze=False,
        inp_refs=[],
        api_name="/get_tts_wav"
    )
    print("gpt-sovits-v2 result:", result)
    return result


def predict_tts(text, model, reference_audio_path=None):
    global client
    print(f"Predicting TTS for {model}")
    # Exceptions: special models that shouldn't be passed to the router
    if model == "csm-1b":
        return predict_csm(text)
    elif model == "playdialog-1.0":
        return predict_playdialog(text)
    elif model == "dia-1.6b":
        return predict_dia(text)
    elif model == "index-tts":
        return predict_index_tts(text, reference_audio_path)
    elif model == "spark-tts":
        return predict_spark_tts(text, reference_audio_path)
    elif model == "cosyvoice-2.0":
        return predict_cosyvoice_tts(text, reference_audio_path)
    elif model == "maskgct":
        return predict_maskgct(text, reference_audio_path)
    elif model == "gpt-sovits-v2":
        return predict_gpt_sovits_v2(text, reference_audio_path)

    if not model in model_mapping:
        raise ValueError(f"Model {model} not found")

    # 构建请求体
    payload = {
        "text": text,
        "provider": model_mapping[model]["provider"],
        "model": model_mapping[model]["model"],
    }
    # 仅支持音色克隆的模型传递参考音色
    supports_reference = model in [
        "styletts2", "eleven-multilingual-v2", "eleven-turbo-v2.5", "eleven-flash-v2.5"
    ]
    if reference_audio_path and supports_reference:
        with open(reference_audio_path, "rb") as f:
            audio_bytes = f.read()
            audio_b64 = base64.b64encode(audio_bytes).decode("utf-8")
        # 不同模型参考音色字段不同
        if model == "styletts2":
            payload["reference_speaker"] = audio_b64
        else:  # elevenlabs 系列
            payload["reference_audio"] = audio_b64

    result = requests.post(
        url,
        headers=headers,
        data=json.dumps(payload),
    )

    response_json = result.json()

    audio_data = response_json["audio_data"]  # base64 encoded audio data
    extension = response_json["extension"]
    # Decode the base64 audio data
    audio_bytes = base64.b64decode(audio_data)

    # Create a temporary file to store the audio data
    with tempfile.NamedTemporaryFile(delete=False, suffix=f".{extension}") as temp_file:
        temp_file.write(audio_bytes)
        temp_path = temp_file.name

    return temp_path


if __name__ == "__main__":
    print(
        predict_dia(
            [
                {"text": "Hello, how are you?", "speaker_id": 0},
                {"text": "I'm great, thank you!", "speaker_id": 1},
            ]
        )
    )
    # print("Predicting PlayDialog")
    # print(
    #     predict_playdialog(
    #         [
    #             {"text": "Hey how are you doing.", "speaker_id": 0},
    #             {"text": "Pretty good, pretty good.", "speaker_id": 1},
    #             {"text": "I'm great, so happy to be speaking to you.", "speaker_id": 0},
    #         ]
    #     )
    # )