Funktioniert
This commit is contained in:
@@ -1,7 +1,8 @@
|
||||
# Qwen3-TTS — Streaming-API (Token-Stream → Audio)
|
||||
|
||||
WebSocket-Endpoint, um Text **inkrementell** (z. B. Token für Token aus einem LLM)
|
||||
zu senden und Audio **satzweise und chunked** zurückzubekommen. Der Server puffert
|
||||
WebSocket-Endpoint, um Text **inkrementell** zu senden und Audio **satzweise und
|
||||
chunked** zurückzubekommen. LLM-Tokens sollen clientseitig zu vollstaendigen
|
||||
Woertern gepuffert werden; der Client splittet nicht selbst in Saetze. Der Server puffert
|
||||
den einströmenden Text, segmentiert ihn an Satz-/Teilsatzgrenzen und synthetisiert
|
||||
jeden Abschnitt, sobald er vollständig ist.
|
||||
|
||||
@@ -37,14 +38,15 @@ Eine WebSocket-Verbindung = eine Session. Ablauf:
|
||||
"split_granularity": "sentence"
|
||||
}
|
||||
```
|
||||
2. **`input.text`** — beliebig viele; je Nachricht ein Token/Fragment/Satz (max. 128 KB je Frame):
|
||||
2. **`input.text`** — beliebig viele; je Nachricht ein vollstaendiges Wort oder Wort mit Satzzeichen (max. 128 KB je Frame):
|
||||
```json
|
||||
{"type": "input.text", "text": "Hallo"}
|
||||
{"type": "input.text", "text": ", wie geht es "}
|
||||
{"type": "input.text", "text": "Ihnen?"}
|
||||
{"type": "input.text", "text": "Hallo "}
|
||||
{"type": "input.text", "text": "Aquantico, "}
|
||||
{"type": "input.text", "text": "willkommen!"}
|
||||
```
|
||||
Der Server hängt alles an einen Puffer an und löst Synthese aus, **sobald eine
|
||||
Satz-/Teilsatzgrenze erkannt wird**. Unvollständige Reste bleiben gepuffert.
|
||||
Der Client soll rohe LLM-Tokens zu Woertern puffern und **keine Saetze selbst
|
||||
splitten**. Der Server haengt alles an einen Puffer an und loest Synthese aus,
|
||||
**sobald eine Satz-/Teilsatzgrenze erkannt wird**. Unvollstaendige Reste bleiben gepuffert.
|
||||
Max. 30 s Pause zwischen zwei Nachrichten (idle timeout).
|
||||
3. **`input.done`** — Eingabe beendet; verbleibender Pufferinhalt wird noch synthetisiert,
|
||||
dann folgt `session.done`.
|
||||
@@ -98,7 +100,7 @@ Eine WebSocket-Verbindung = eine Session. Ablauf:
|
||||
## Python-Beispiel (Token-Stream → PCM)
|
||||
|
||||
```python
|
||||
import asyncio, json, ssl, inspect, websockets
|
||||
import asyncio, json, ssl, inspect, re, websockets
|
||||
|
||||
WSS = "wss://qwen3-tts.aquantico.de/v1/audio/speech/stream"
|
||||
TOKEN = "<BEARER_TOKEN>"
|
||||
@@ -131,11 +133,24 @@ async def speak(token_iter):
|
||||
if msg["type"] == "error": raise RuntimeError(msg["message"])
|
||||
|
||||
rt = asyncio.create_task(reader())
|
||||
for tok in token_iter: # Tokens direkt aus dem LLM
|
||||
await ws.send(json.dumps({"type": "input.text", "text": tok}))
|
||||
async for word in words_from_tokens(token_iter):
|
||||
await ws.send(json.dumps({"type": "input.text", "text": word}))
|
||||
await ws.send(json.dumps({"type": "input.done"}))
|
||||
await rt
|
||||
|
||||
async def words_from_tokens(token_iter):
|
||||
buf = ""
|
||||
for tok in token_iter:
|
||||
buf += tok
|
||||
while True:
|
||||
m = re.match(r"(\S+\s+)(.*)", buf)
|
||||
if not m:
|
||||
break
|
||||
yield m.group(1)
|
||||
buf = m.group(2)
|
||||
if buf:
|
||||
yield buf
|
||||
|
||||
def handle_pcm(chunk: bytes):
|
||||
... # z.B. sounddevice / Datei / Web-Client
|
||||
```
|
||||
@@ -145,3 +160,17 @@ def handle_pcm(chunk: bytes):
|
||||
- `session.config` muss innerhalb **10 s** kommen, sonst Abbruch.
|
||||
- Max. **30 s** Pause zwischen Nachrichten (idle timeout).
|
||||
- `session.config` ≤ 4 MB (für große `ref_audio`-Payloads), `input.text` ≤ 128 KB pro Frame.
|
||||
|
||||
|
||||
## WebSocket-Logging
|
||||
|
||||
Fuer Diagnose kann der Logging-Proxy verwendet werden:
|
||||
|
||||
```text
|
||||
ws://localhost:8094/v1/audio/speech/stream
|
||||
```
|
||||
|
||||
Er leitet an `ws://localhost:8091/v1/audio/speech/stream` weiter und schreibt
|
||||
`session.config`, `input.text`, JSON-Events und Audio-Chunk-Groessen in die
|
||||
Containerlogs (`docker logs qwen3-tts`). So ist sichtbar, ob `seed`, `voice`,
|
||||
`language` und `instructions` tatsaechlich im WebSocket-Call gesendet werden.
|
||||
|
||||
24
clone_model.sh
Executable file
24
clone_model.sh
Executable file
@@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
case "${1:-}" in
|
||||
load|start)
|
||||
docker compose --profile clone up -d qwen3-tts-clone
|
||||
;;
|
||||
unload|stop)
|
||||
docker compose --profile clone stop qwen3-tts-clone >/dev/null
|
||||
docker compose --profile clone rm -f qwen3-tts-clone >/dev/null
|
||||
;;
|
||||
status)
|
||||
docker ps --filter "name=qwen3-tts-clone" --format '{{.Names}} {{.Status}} {{.Ports}}'
|
||||
;;
|
||||
logs)
|
||||
docker logs -f qwen3-tts-clone
|
||||
;;
|
||||
*)
|
||||
echo "Usage: $0 {load|unload|status|logs}" >&2
|
||||
echo "Set QWEN3_TTS_CLONE_MODEL=<model-id> before load if needed." >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
@@ -8,12 +8,26 @@ services:
|
||||
- NVIDIA_DRIVER_CAPABILITIES=compute,utility
|
||||
- VLLM_LOGGING_LEVEL=INFO
|
||||
- HF_HOME=/root/.cache/huggingface
|
||||
# Modelle liegen vollstaendig im Cache → keine Online-Checks gegen huggingface.co.
|
||||
# Verhindert Crash "[Errno -3] Temporary failure in name resolution" bei DNS-Ausfall.
|
||||
- HF_HUB_OFFLINE=1
|
||||
- TRANSFORMERS_OFFLINE=1
|
||||
# Keine Lead-in-Stille (war nur Mitigation; Anfangsstottern ist client-seitig).
|
||||
# Trailing-Stille bleibt per Default (300ms) gegen abruptes Ende.
|
||||
- QWEN3_TTS_LEADIN_SILENCE_MS=0
|
||||
- QWEN3_TTS_MODEL=${QWEN3_TTS_MODEL:-Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice}
|
||||
ports:
|
||||
- "8091:8091"
|
||||
- "8092:8092"
|
||||
- "8094:8094"
|
||||
volumes:
|
||||
- /home/guru/vllm/data/root:/root
|
||||
- ./deploy/qwen3_tts.yaml:/deploy/qwen3_tts.yaml:ro
|
||||
- ./entrypoint.sh:/entrypoint.sh:ro
|
||||
- ./voice_clone_ui.py:/voice_clone_ui.py:ro
|
||||
- ./ws_log_proxy.py:/ws_log_proxy.py:ro
|
||||
- ./patch_qwen3_tts_runtime.py:/patch_qwen3_tts_runtime.py:ro
|
||||
- ./ui:/ui:ro
|
||||
command: ["/bin/bash", "/entrypoint.sh"]
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
@@ -22,3 +36,34 @@ services:
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 120s
|
||||
|
||||
|
||||
qwen3-tts-clone:
|
||||
image: vllm/vllm-omni:latest-aarch64
|
||||
container_name: qwen3-tts-clone
|
||||
runtime: nvidia
|
||||
profiles:
|
||||
- clone
|
||||
environment:
|
||||
- NVIDIA_VISIBLE_DEVICES=all
|
||||
- NVIDIA_DRIVER_CAPABILITIES=compute,utility
|
||||
- VLLM_LOGGING_LEVEL=INFO
|
||||
- HF_HOME=/root/.cache/huggingface
|
||||
- HF_HUB_OFFLINE=1
|
||||
- TRANSFORMERS_OFFLINE=1
|
||||
- QWEN3_TTS_CLONE_MODEL=${QWEN3_TTS_CLONE_MODEL:-}
|
||||
- QWEN3_TTS_CLONE_PORT=8093
|
||||
ports:
|
||||
- "8093:8093"
|
||||
volumes:
|
||||
- /home/guru/vllm/data/root:/root
|
||||
- ./deploy/qwen3_tts.yaml:/deploy/qwen3_tts.yaml:ro
|
||||
- ./entrypoint_clone.sh:/entrypoint_clone.sh:ro
|
||||
command: ["/bin/bash", "/entrypoint_clone.sh"]
|
||||
restart: "no"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-sf", "http://localhost:8093/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 120s
|
||||
|
||||
233
docs/PROJECT_OVERVIEW.md
Normal file
233
docs/PROJECT_OVERVIEW.md
Normal file
@@ -0,0 +1,233 @@
|
||||
# Projektueberblick: Qwen3-TTS
|
||||
|
||||
Dieses Repository beschreibt und betreibt einen Qwen3-TTS-Service auf Basis von
|
||||
`vllm/vllm-omni`. Es enthaelt die Containerkonfiguration, Startlogik,
|
||||
Deployment-Parameter und einfache Testclients fuer REST- und WebSocket-Zugriffe.
|
||||
Der eigentliche Modellserver kommt aus dem vLLM-Omni-Image; dieses Repo kapselt
|
||||
vor allem Betrieb, Konfiguration und API-Nutzung.
|
||||
|
||||
## Zweck
|
||||
|
||||
Der Service stellt Text-to-Speech auf Port `8091` bereit. Standard ist
|
||||
`Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice`; fuer dialogweite feste
|
||||
geklonte Stimmen wird `Qwen/Qwen3-TTS-12Hz-1.7B-Base` gestartet. Er unterstuetzt
|
||||
zwei Nutzungsarten:
|
||||
|
||||
- REST-Vollausgabe ueber `POST /v1/audio/speech`
|
||||
- inkrementelles Streaming ueber `WS /v1/audio/speech/stream`
|
||||
|
||||
Das Streaming-Protokoll ist fuer Token-Streams aus LLMs gedacht: Textfragmente
|
||||
werden fortlaufend gesendet, der Server segmentiert an Satz- oder
|
||||
Teilsatzgrenzen und gibt Audio pro Segment zurueck.
|
||||
|
||||
## Architektur
|
||||
|
||||
Der Container startet `vllm-omni serve` mit Omni-Modus und einer
|
||||
Deployment-Konfiguration aus `deploy/qwen3_tts.yaml`. Die Pipeline besteht aus
|
||||
zwei Stages:
|
||||
|
||||
- Stage 0: Talker-LLM fuer die sprachliche Token-/Codec-Erzeugung
|
||||
- Stage 1: Code2Wav-Decoder fuer die Audiosynthese
|
||||
|
||||
Beide Stages laufen auf GPU `0` und kommunizieren ueber einen
|
||||
Shared-Memory-Connector. `async_chunk: true` und `codec_streaming: true`
|
||||
aktivieren chunked Audioausgabe. Die Konfiguration ist auf eine geteilte
|
||||
DGX-Spark-Maschine mit begrenzter GPU-Speichernutzung ausgelegt
|
||||
(`gpu_memory_utilization: 0.08` pro Stage).
|
||||
|
||||
## Wichtige Dateien
|
||||
|
||||
- `docker-compose.yml`: definiert den Service `qwen3-tts`, nutzt
|
||||
`vllm/vllm-omni:latest-aarch64`, bindet Port `8091` fuer die API und `8092`
|
||||
fuer die Browser-Oberflaeche, mountet Hugging-Face-Daten sowie Deployment-,
|
||||
Entrypoint- und UI-Dateien.
|
||||
- `entrypoint.sh`: startet die Voice-Cloning-Oberflaeche im Container, startet
|
||||
den vLLM-Omni-Server und fuehrt nach erfolgreichem Healthcheck einen
|
||||
Warmup-Request aus, damit CUDA-Graph-/Compile-Kosten nicht den ersten echten
|
||||
Nutzerrequest treffen.
|
||||
- `start.sh`: startet den Compose-Service, filtert relevante Containerlogs in
|
||||
lesbare Statusmeldungen und wartet auf `/health`.
|
||||
- `deploy/qwen3_tts.yaml`: aktive Deployment-Konfiguration fuer die zweistufige
|
||||
TTS-Pipeline, Streaming-Connectoren, Sampling-Parameter und Speicherlimits.
|
||||
- `deploy/qwen3_tts.streaming.yaml`: aktuell identisch zur aktiven
|
||||
Deployment-Konfiguration; offenbar als explizite Streaming-Variante abgelegt.
|
||||
- `STREAMING_API.md`: detaillierte API-Dokumentation fuer REST, WebSocket,
|
||||
Session-Protokoll, Audioformate, Segmentierung und bekannte Latenzwerte.
|
||||
- `test_http.py`: einfacher REST-Test gegen `http://localhost:8091`, schreibt
|
||||
`test_output.wav`.
|
||||
- `test_ws.py`: WebSocket-Streaming-Test gegen
|
||||
`ws://localhost:8091/v1/audio/speech/stream`, sammelt PCM-Chunks und schreibt
|
||||
`test_stream_output.wav`.
|
||||
- `ui/voice-cloning.html`: einfache Browser-Oberflaeche fuer Voice Cloning mit
|
||||
Referenzaudio, Referenztranskript, Zieltext und Ausgabeplayer.
|
||||
- `voice_clone_ui.py`: kleiner lokaler Webserver, der die HTML-Oberflaeche
|
||||
ausliefert und REST-Anfragen als Same-Origin-Proxy an den TTS-Service
|
||||
weiterleitet.
|
||||
|
||||
## Betrieb
|
||||
|
||||
Der normale TTS/UI-Service wird gestartet und gestoppt ueber:
|
||||
|
||||
```bash
|
||||
./start_tts.sh
|
||||
./stop_tts.sh
|
||||
```
|
||||
|
||||
`./start.sh` bleibt als kompatibler Wrapper auf `./start_tts.sh` erhalten. Beim
|
||||
Start des TTS-Service wird ein eventuell laufender Clone-Container automatisch
|
||||
gestoppt. Das Skript startet danach `qwen3-tts`, beobachtet die Logs und beendet
|
||||
sich erfolgreich, sobald `http://localhost:8091/health` antwortet. Die UI ist
|
||||
unter `http://localhost:8092/` erreichbar. Der WebSocket-Logging-Proxy laeuft auf `ws://localhost:8094/v1/audio/speech/stream`.
|
||||
|
||||
Voraussetzungen sind ein Docker-Setup mit NVIDIA-Runtime, Zugriff auf GPU `0`
|
||||
und ein lokaler Modell-/Cache-Mount unter `/home/guru/vllm/data/root`, der in den
|
||||
Container als `/root` eingebunden wird.
|
||||
|
||||
## Dialogbetrieb mit fester geklonter Stimme
|
||||
|
||||
Die Qwen-Hugging-Face-Modelcard listet `Qwen/Qwen3-TTS-12Hz-1.7B-Base`
|
||||
als Base-Modell fuer 3-Sekunden-Voice-Cloning und beschreibt wiederverwendbare
|
||||
Voice-Clone-Prompts. Im lokalen vLLM-Omni-Code wird ein gespeicherter oder aus
|
||||
`ref_audio` extrahierter Speaker-Anker als `ref_spk_embedding` an Qwen3-TTS
|
||||
uebergeben. Fuer einen Voice-Dialog ist deshalb der richtige Betriebsmodus:
|
||||
|
||||
```bash
|
||||
./download_qwen3_tts_base.sh # einmalig, falls das Base-Modell noch fehlt
|
||||
./start_dialog_tts.sh # startet qwen3-tts mit Qwen3-TTS-12Hz-1.7B-Base
|
||||
```
|
||||
|
||||
Danach unter `http://localhost:8092/` die Referenzstimme speichern und fuer
|
||||
Ausgaben `Gespeicherte Stimme` mit Task `Base` verwenden. Der Server nutzt dann
|
||||
bei jeder Synthese dieselbe gespeicherte Referenzstimme; intern wird daraus der
|
||||
Qwen3-TTS-Speaker-Anker erzeugt und gecacht. Fuer harte Persistenz ohne
|
||||
Referenzaudio kann alternativ ein passender 2048-dim `speaker_embedding` ueber
|
||||
`/api/voice-vectors` gespeichert und bei jeder Ausgabe mitgesendet werden.
|
||||
|
||||
`seed` bleibt zusaetzlich sinnvoll, ersetzt aber keinen Speaker-Anker. Der
|
||||
Runtime-Patch `patch_qwen3_tts_runtime.py` sorgt dafuer, dass `seed` im
|
||||
WebSocket-Pfad tatsaechlich an jedes Segment weitergereicht wird und die
|
||||
relevanten Werte im Containerlog sichtbar sind.
|
||||
|
||||
## API-Nutzung
|
||||
|
||||
REST eignet sich fuer komplette Eingabetexte und liefert standardmaessig eine
|
||||
WAV-Antwort. Der Testclient nutzt:
|
||||
|
||||
```bash
|
||||
python3 test_http.py
|
||||
```
|
||||
|
||||
WebSocket eignet sich fuer Low-Latency-Szenarien, in denen Text schrittweise
|
||||
eintrifft. Eine Session beginnt mit `session.config`, nimmt danach beliebig viele
|
||||
`input.text`-Frames an und wird mit `input.done` abgeschlossen. Audio kommt als
|
||||
PCM-Binaerframes oder WAV-Bloecke zurueck. Der Testclient nutzt:
|
||||
|
||||
```bash
|
||||
python3 test_ws.py
|
||||
```
|
||||
|
||||
Details zu Nachrichtentypen, Timeouts, Audioformaten und externem Zugriff stehen
|
||||
in `STREAMING_API.md`.
|
||||
|
||||
## Voice-Cloning-Oberflaeche
|
||||
|
||||
Die Browser-Oberflaeche laeuft im selben Container wie der TTS-Service. Beim
|
||||
Start ueber `./start.sh` wird neben der API auf Port `8091` auch die UI auf Port
|
||||
`8092` veroeffentlicht:
|
||||
|
||||
```text
|
||||
http://localhost:8092/
|
||||
```
|
||||
|
||||
Der im Container gestartete Proxy leitet `POST /api/v1/audio/speech` intern an
|
||||
`http://localhost:8091/v1/audio/speech` weiter. Dadurch muss der Browser keine
|
||||
direkten Cross-Origin-Requests an die TTS-API senden. Zusaetzlich verwaltet der
|
||||
Proxy persistente Speaker-Vektoren unter `/root/.cache/qwen3-tts-ui/vectors`
|
||||
und stellt dafuer diese lokalen Routen bereit:
|
||||
|
||||
- `GET /api/voice-vectors`
|
||||
- `POST /api/voice-vectors` mit `name`, `description`, `embedding`
|
||||
- `GET /api/voice-vectors/{name}`
|
||||
- `DELETE /api/voice-vectors/{name}`
|
||||
- `POST /api/voice-vectors/{name}/speech` fuer Synthese mit `speaker_embedding`
|
||||
|
||||
|
||||
Die Oberflaeche unterstuetzt zwei Testpfade:
|
||||
|
||||
- `Gespeicherte Stimme`: listet Stimmen ueber `GET /api/v1/audio/voices`,
|
||||
erzeugt Testaudio mit `voice: <name>` und kann hochgeladene Stimmen loeschen.
|
||||
- `Ad-hoc Referenz`: sendet `ref_audio` und `ref_text` direkt im Speech-Request,
|
||||
ohne die Stimme dauerhaft zu speichern.
|
||||
|
||||
Neue Stimmen koennen in der UI mit Referenzaudio, Referenztranskript, Name,
|
||||
Consent-ID und Beschreibung ueber `POST /api/v1/audio/voices` gespeichert
|
||||
werden. Der Proxy leitet diese Route im Container an `/v1/audio/voices` weiter.
|
||||
|
||||
`ref_audio` kann beim Ad-hoc-Test als Data-URL oder reine Base64-Zeichenkette
|
||||
gesendet werden. Welche Variante akzeptiert wird, haengt vom
|
||||
vLLM-Omni/Qwen3-TTS-API-Schema des laufenden Images ab. Falls der aktuell
|
||||
gestartete Modellpfad `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` kein `Base`-Cloning
|
||||
unterstuetzt, meldet die API einen Fehler; dann muss im UI-Feld `Modell` ein
|
||||
klonfaehiges Qwen3-TTS-Modell eingetragen und der Container entsprechend
|
||||
gestartet werden.
|
||||
|
||||
|
||||
## Optionaler Clone-Service
|
||||
|
||||
Fuer die Trennung von produktiver TTS-Ausgabe und Voice-Cloning gibt es einen
|
||||
optionalen zweiten Container `qwen3-tts-clone` im Compose-Profil `clone`. Er
|
||||
laeuft auf Port `8093` und wird nur bei Bedarf gestartet:
|
||||
|
||||
```bash
|
||||
QWEN3_TTS_CLONE_MODEL=<lokal-verfuegbares-clone/base-modell> ./start_clone.sh
|
||||
```
|
||||
|
||||
Beim Start des Clone-Service wird der normale TTS/UI-Container automatisch
|
||||
gestoppt. Damit laufen TTS und Clone nicht gleichzeitig auf derselben GPU.
|
||||
`clone_model.sh` bleibt fuer manuelles `status`, `logs` und `unload` erhalten:
|
||||
|
||||
```bash
|
||||
./stop_clone.sh
|
||||
./clone_model.sh status
|
||||
./clone_model.sh logs
|
||||
./clone_model.sh unload
|
||||
```
|
||||
|
||||
Default fuer den Clone-Service ist jetzt `Qwen/Qwen3-TTS-12Hz-1.7B-Base`.
|
||||
Wenn das Modell noch nicht im lokalen Hugging-Face-Cache liegt, zuerst
|
||||
`./download_qwen3_tts_base.sh` ausfuehren oder dem Container funktionierenden
|
||||
Internet-/DNS-Zugriff geben.
|
||||
|
||||
Der UI-Proxy kennt dafuer diese optionalen Routen:
|
||||
|
||||
- `GET /api/clone/health` -> `qwen3-tts-clone:8093/health`
|
||||
- `POST /api/clone/speech` -> `qwen3-tts-clone:8093/v1/audio/speech`
|
||||
|
||||
Die normale TTS-Ausgabe kann bereits gespeicherte Speaker-Vektoren verwenden.
|
||||
Dafuer sendet `POST /api/voice-vectors/{name}/speech` den gespeicherten Vektor
|
||||
als `speaker_embedding` mit `task_type: Base` und `x_vector_only_mode: true` an
|
||||
den Haupt-TTS-Service.
|
||||
|
||||
## Bekannte Eigenschaften und Risiken
|
||||
|
||||
- Das Repo enthaelt keine eigene Serverimplementierung; Verhalten und API werden
|
||||
vom verwendeten vLLM-Omni-Image und Modellcode bestimmt.
|
||||
- Die Compose-Konfiguration pinnt das Image nicht auf einen Digest. Updates von
|
||||
`vllm/vllm-omni:latest-aarch64` koennen Verhalten oder Kompatibilitaet aendern.
|
||||
- `deploy/qwen3_tts.yaml` und `deploy/qwen3_tts.streaming.yaml` sind derzeit
|
||||
doppelt vorhanden. Wenn beide Varianten dauerhaft gebraucht werden, sollte der
|
||||
Unterschied dokumentiert oder eine Datei entfernt werden.
|
||||
- `test_ws.py` installiert `websockets` bei fehlendem Import automatisch per pip.
|
||||
Das ist praktisch fuer lokale Tests, aber fuer reproduzierbare Umgebungen
|
||||
weniger kontrolliert als eine explizite Requirements-Datei.
|
||||
- Die Voice-Cloning-Oberflaeche ist ein Client fuer vorhandene API-Felder. Sie
|
||||
extrahiert oder trainiert keine Sprechervektoren selbst.
|
||||
- Die Streaming-Doku weist auf moegliche Stimm-Drift zwischen Segmenten hin, weil
|
||||
jedes Segment separat synthetisiert wird.
|
||||
|
||||
## Kurzfazit
|
||||
|
||||
Das Projekt ist ein schlankes Betriebs- und Integrationsrepo fuer einen
|
||||
Qwen3-TTS-Service. Sein Schwerpunkt liegt auf GPU-Deployment, Warmup,
|
||||
Low-Latency-Streaming und pruefbaren Clientbeispielen, nicht auf eigener
|
||||
Modellentwicklung oder einer selbst implementierten API-Schicht.
|
||||
220
docs/VOICE_CLONING.md
Normal file
220
docs/VOICE_CLONING.md
Normal file
@@ -0,0 +1,220 @@
|
||||
# Voice Cloning — Qwen3-TTS
|
||||
|
||||
Vollständige Übersicht, wie in diesem Projekt Stimmen geklont werden: welche
|
||||
Modelle das können, welcher Container/Port wofür zuständig ist, welche
|
||||
API-Felder und UI-Pfade es gibt und welche Fallstricke (Embedding-Dimensionen,
|
||||
Stimm-Drift) zu beachten sind.
|
||||
|
||||
> Stand: 2026-06-17. Quelle: Code in diesem Repo (`docker-compose.yml`,
|
||||
> `entrypoint*.sh`, `voice_clone_ui.py`, `ui/voice-cloning.html`,
|
||||
> `patch_qwen3_tts_runtime.py`, `start_*.sh`) + `docs/PROJECT_OVERVIEW.md`.
|
||||
|
||||
---
|
||||
|
||||
## TL;DR
|
||||
|
||||
- **Cloning braucht das Base-Modell**, nicht CustomVoice.
|
||||
- `Qwen/Qwen3-TTS-12Hz-1.7B-Base` → hat Speaker-Encoder, kann aus 3 s
|
||||
Referenzaudio klonen.
|
||||
- `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` (Default-Betrieb, Port 8091) → hat
|
||||
**keinen** Speaker-Encoder (`speaker_encoder_config = None`), nur 9 feste
|
||||
Sprecher. Klon-Versuch crasht mit `Expected size 2048 but got size 1024`.
|
||||
- **Beide Modelle liegen bereits im lokalen HF-Cache** unter
|
||||
`/home/guru/vllm/data/root/.cache/huggingface/hub/`.
|
||||
- **Klon-Workflow:** Base-Modell starten → in der Browser-UI (Port 8092)
|
||||
Referenzaudio + Transkript hochladen → testen oder als Stimme speichern.
|
||||
- **Für stabile Dialogstimmen:** Referenz-WAV bzw. fester Speaker-Vektor
|
||||
(2048-dim) muss bei *jedem* Turn mitgegeben werden, sonst driftet das Timbre.
|
||||
|
||||
---
|
||||
|
||||
## Modellfamilie & Cloning-Fähigkeit
|
||||
|
||||
| Modell | Port/Betrieb | Cloning? | Wie |
|
||||
|---|---|---|---|
|
||||
| `Qwen3-TTS-12Hz-1.7B-CustomVoice` | Default, Port 8091 | ❌ Nein | 9 feste Sprecher (Vivian/Serena/Uncle_Fu/Dylan/Eric/Ryan/Aiden/Ono_Anna/Sohee), `instruct` steuert nur Emotion/Stil |
|
||||
| `Qwen3-TTS-12Hz-1.7B-Base` | via `start_dialog_tts.sh` (Port 8091) oder Clone-Service (Port 8093) | ✅ Ja | 3-Sekunden-Clone aus `ref_audio`, Speaker-Encoder vorhanden |
|
||||
| `Qwen3-TTS-12Hz-1.7B-VoiceDesign` | nicht eingerichtet | ⚠️ Indirekt | erzeugt neue Stimme aus Textbeschreibung (`instruct`), kein Referenzaudio |
|
||||
|
||||
**Embedding-Dimension:** 2048 für die 1.7B-Modelle (1024 für 0.6B-Varianten).
|
||||
Das ist die Ursache des `Expected size 2048 but got size 1024`-Crashs, wenn man
|
||||
CustomVoice ein Clone-Embedding unterschieben will.
|
||||
|
||||
---
|
||||
|
||||
## Betriebsmodi / Container
|
||||
|
||||
Definiert in `docker-compose.yml`. Beide Container teilen sich GPU 0 und den
|
||||
HF-Cache (`/home/guru/vllm/data/root:/root`), laufen aber **nicht gleichzeitig**
|
||||
(Start-Skripte stoppen jeweils den anderen).
|
||||
|
||||
### 1. Haupt-Container `qwen3-tts` (Port 8091/8092/8094)
|
||||
- Startet via `entrypoint.sh`: `vllm-omni serve`, plus
|
||||
- Browser-UI (`voice_clone_ui.py`) auf **8092**
|
||||
- WS-Logging-Proxy (`ws_log_proxy.py`) auf **8094**
|
||||
- Runtime-Patch (`patch_qwen3_tts_runtime.py`) für Seed-Propagation im WS-Pfad
|
||||
- Auto-Warmup-Request nach `/health`
|
||||
- Modell wählbar über `QWEN3_TTS_MODEL`:
|
||||
- `./start_custom_tts.sh` → CustomVoice (kein Cloning, schnell, Standard)
|
||||
- `./start_dialog_tts.sh` → **Base** (Cloning + feste Dialogstimmen)
|
||||
- `./start_tts.sh` / `./start.sh` → respektiert `QWEN3_TTS_MODEL`, Default CustomVoice
|
||||
- `./stop_tts.sh` → stoppt + entfernt den Container
|
||||
|
||||
### 2. Optionaler Clone-Container `qwen3-tts-clone` (Port 8093, Compose-Profil `clone`)
|
||||
- Trennt Cloning von produktiver Ausgabe. Eigenes `entrypoint_clone.sh`,
|
||||
`gpu-memory-utilization 0.10`, `restart: "no"`.
|
||||
- Start: `QWEN3_TTS_CLONE_MODEL=<modell> ./start_clone.sh`
|
||||
(Default-Modell: `Qwen3-TTS-12Hz-1.7B-Base`). Stoppt automatisch `qwen3-tts`.
|
||||
- Verwaltung: `./clone_model.sh {load|unload|status|logs}`, `./stop_clone.sh`.
|
||||
- UI-Proxy-Routen dafür: `GET /api/clone/health`, `POST /api/clone/speech`.
|
||||
|
||||
### Modell-Download (einmalig)
|
||||
```bash
|
||||
./download_qwen3_tts_base.sh # lädt Qwen3-TTS-12Hz-1.7B-Base in den Cache
|
||||
```
|
||||
(Beide Modelle sind aktuell bereits im Cache vorhanden.)
|
||||
|
||||
---
|
||||
|
||||
## Klon-Workflow (empfohlen, über die UI)
|
||||
|
||||
```bash
|
||||
./download_qwen3_tts_base.sh # nur falls Base noch nicht im Cache
|
||||
./start_dialog_tts.sh # startet qwen3-tts mit dem Base-Modell
|
||||
# Browser: http://localhost:8092/
|
||||
```
|
||||
|
||||
In der UI (`ui/voice-cloning.html`) gibt es zwei Klon-Pfade plus Vektor-Verwaltung:
|
||||
|
||||
1. **Ad-hoc Referenz** — Referenzaudio + Transkript direkt im Request mitsenden,
|
||||
Stimme wird nicht gespeichert. Sendet `task_type="Base"`, `ref_audio`
|
||||
(Base64 oder Data-URL), `ref_text`.
|
||||
2. **Gespeicherte Stimme** — Stimme dauerhaft registrieren via
|
||||
`POST /v1/audio/voices` (multipart: audio, ref_text, name, consent,
|
||||
description), danach Synthese mit `voice=<name>`. Auflisten über
|
||||
`GET /v1/audio/voices`, löschen via DELETE.
|
||||
3. **Speaker-Vektoren** — persistente 2048-dim Embeddings, lokal verwaltet vom
|
||||
UI-Proxy unter `/root/.cache/qwen3-tts-ui/vectors`. Synthese mit fixem Vektor
|
||||
über `POST /api/voice-vectors/{name}/speech` (setzt intern `task_type="Base"`,
|
||||
`x_vector_only_mode=true`, `speaker_embedding=<vektor>`).
|
||||
|
||||
> Wichtig: Die UI/der Proxy **extrahiert selbst keine Sprechervektoren** und
|
||||
> trainiert nichts. Er ist ein Client für die vom Modell bereitgestellten
|
||||
> API-Felder. Das Berechnen des x-vectors macht der Modellserver (nur Base).
|
||||
|
||||
---
|
||||
|
||||
## REST-API-Felder für Cloning
|
||||
|
||||
`POST /v1/audio/speech` (Beispiel Ad-hoc-Clone):
|
||||
```json
|
||||
{
|
||||
"model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base",
|
||||
"input": "Der zu sprechende Zieltext.",
|
||||
"task_type": "Base",
|
||||
"ref_audio": "<base64 oder data-URL der Referenz-WAV>",
|
||||
"ref_text": "Transkript der Referenzaufnahme (ICL)",
|
||||
"language": "German",
|
||||
"response_format": "wav"
|
||||
}
|
||||
```
|
||||
|
||||
Synthese mit gespeichertem Speaker-Vektor (vom UI-Proxy erzeugt):
|
||||
```json
|
||||
{
|
||||
"task_type": "Base",
|
||||
"x_vector_only_mode": true,
|
||||
"speaker_embedding": [ /* 2048 floats */ ],
|
||||
"input": "...", "language": "German"
|
||||
}
|
||||
```
|
||||
|
||||
Voice registrieren: `POST /v1/audio/voices` (multipart) — Felder `audio_sample`
|
||||
**oder** `speaker_embedding` (2048-dim), `consent`, `name`, optional `ref_text` /
|
||||
`speaker_description`. `GET /v1/audio/voices` listet, `DELETE
|
||||
/v1/audio/voices/{name}` löscht.
|
||||
|
||||
---
|
||||
|
||||
## UI-Proxy-Routen (`voice_clone_ui.py`, Port 8092)
|
||||
|
||||
Same-Origin-Proxy, vermeidet CORS im Browser:
|
||||
|
||||
| Route | Ziel / Funktion |
|
||||
|---|---|
|
||||
| `GET /` | liefert `ui/voice-cloning.html` |
|
||||
| `GET /health` | → `8091/health` |
|
||||
| `GET /api/config` | aktuelles Modell |
|
||||
| `GET/POST /api/v1/audio/voices` | → `8091/v1/audio/voices` (Liste / registrieren) |
|
||||
| `DELETE /api/v1/audio/voices/{name}` | Voice löschen |
|
||||
| `POST /api/v1/audio/speech` | → `8091/v1/audio/speech` (Synthese) |
|
||||
| `GET/POST /api/voice-vectors` | lokale Speaker-Vektoren auflisten / speichern |
|
||||
| `GET/DELETE /api/voice-vectors/{name}` | einzelnen Vektor holen / löschen |
|
||||
| `POST /api/voice-vectors/{name}/speech` | Synthese mit gespeichertem Vektor |
|
||||
| `GET /api/clone/health` | → `qwen3-tts-clone:8093/health` |
|
||||
| `POST /api/clone/speech` | → `qwen3-tts-clone:8093/v1/audio/speech` |
|
||||
|
||||
Vektor-Speicher: `/root/.cache/qwen3-tts-ui/vectors/<name>.json`, Namen müssen
|
||||
`^[A-Za-z0-9_.-]{1,80}$` matchen, Embedding ≤ 4096 finite Floats.
|
||||
|
||||
---
|
||||
|
||||
## Stimm-Persistenz & Drift (wichtig für Dialoge)
|
||||
|
||||
Eine geklonte/designte Stimme ist **nicht automatisch über mehrere Aufrufe
|
||||
stabil**. Jeder getrennte Call zieht die Sprecher-Realisierung neu aus der
|
||||
Verteilung → Timbre/Tonhöhe driften über Dialog-Turns.
|
||||
|
||||
- Gemessen (CustomVoice, voice=Ryan, seed=42, getrennte Calls): F0 schwankt
|
||||
128–169 Hz (~11 %), Spektralzentroid ~11 %.
|
||||
- **Temperatur/top_k senken hilft nicht** — Drift kommt nicht vom Sampling.
|
||||
- `seed` allein reicht nicht (ersetzt keinen Speaker-Anker), wird aber im
|
||||
WS-Pfad jetzt korrekt pro Segment durchgereicht (`patch_qwen3_tts_runtime.py`).
|
||||
|
||||
**Lösungen:**
|
||||
1. **Fester Speaker-Anker (eigentliche Lösung):** Referenz-WAV bzw. 2048-dim
|
||||
`speaker_embedding` bei *jedem* Turn mitgeben (Base-Pfad). So bleibt das
|
||||
Timbre konstant. Genau dafür sind die gespeicherten Voice-Vektoren da.
|
||||
2. **Sofort-Mitigation (ohne Base):** mehr Text pro Call bündeln (2–4 Sätze) —
|
||||
Drift entsteht *zwischen* Calls, nicht innerhalb. RTF ~0.4, TTFA ~0.16 s
|
||||
erlauben Bündeln ohne große Latenzkosten.
|
||||
|
||||
**„Design-once-then-fixed-voice"-Workflow:** 1× mit VoiceDesign eine Referenz-WAV
|
||||
erzeugen → diese WAV als `ref_audio` mit dem Base-Modell für alle Dialog-Turns
|
||||
nutzen. Braucht VoiceDesign UND Base.
|
||||
|
||||
---
|
||||
|
||||
## Fallstricke
|
||||
|
||||
- **CustomVoice + Clone = Crash:** Server extrahiert 1024-dim, Modell erwartet
|
||||
2048-dim → `RuntimeError: Expected size 2048 but got size 1024`. Container
|
||||
fängt sich via `restart: unless-stopped` + Warmup in Sekunden. Die UI fängt
|
||||
diesen Fall vorab ab und zeigt eine erklärende Fehlermeldung.
|
||||
- Für Cloning **muss** das Base-Modell laufen (`start_dialog_tts.sh` oder
|
||||
Clone-Service), nicht der Default-CustomVoice-Betrieb.
|
||||
- Image ist nicht auf Digest gepinnt (`vllm/vllm-omni:latest-aarch64`); Updates
|
||||
können API-Verhalten ändern.
|
||||
- `ref_audio` als Data-URL **oder** reine Base64 — was akzeptiert wird, hängt
|
||||
vom API-Schema des laufenden Images ab.
|
||||
|
||||
---
|
||||
|
||||
## Schnellreferenz Befehle
|
||||
|
||||
```bash
|
||||
# Cloning-fähigen Betrieb starten (Base-Modell)
|
||||
./start_dialog_tts.sh
|
||||
# UI: http://localhost:8092/ API: http://localhost:8091
|
||||
|
||||
# Zurück auf schnellen Standardbetrieb (keine Klone)
|
||||
./start_custom_tts.sh
|
||||
|
||||
# Getrennter Clone-Service auf Port 8093
|
||||
QWEN3_TTS_CLONE_MODEL=Qwen/Qwen3-TTS-12Hz-1.7B-Base ./start_clone.sh
|
||||
./clone_model.sh status|logs|unload
|
||||
./stop_clone.sh
|
||||
|
||||
# Base-Modell herunterladen (einmalig)
|
||||
./download_qwen3_tts_base.sh
|
||||
```
|
||||
15
download_qwen3_tts_base.sh
Executable file
15
download_qwen3_tts_base.sh
Executable file
@@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
MODEL="${QWEN3_TTS_BASE_MODEL:-Qwen/Qwen3-TTS-12Hz-1.7B-Base}"
|
||||
CACHE_ROOT="/home/guru/vllm/data/root"
|
||||
mkdir -p "$CACHE_ROOT/.cache/huggingface"
|
||||
|
||||
printf "Lade %s in gemeinsamen Container-Cache...\n" "$MODEL"
|
||||
docker run --rm \
|
||||
-e HF_HOME=/root/.cache/huggingface \
|
||||
-v "$CACHE_ROOT:/root" \
|
||||
vllm/vllm-omni:latest-aarch64 \
|
||||
python3 -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='$MODEL')"
|
||||
|
||||
printf "Fertig. Danach: ./start_dialog_tts.sh\n"
|
||||
@@ -6,8 +6,37 @@
|
||||
# (Reboot, Crash-Restart, manuell).
|
||||
set -e
|
||||
|
||||
MODEL="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
|
||||
MODEL="${QWEN3_TTS_MODEL:-Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice}"
|
||||
PORT=8091
|
||||
UI_PORT=8092
|
||||
WS_LOG_PORT=8094
|
||||
|
||||
# --- Runtime-Patches fuer Qwen3-TTS anwenden ---
|
||||
if [ -f /patch_qwen3_tts_runtime.py ]; then
|
||||
python3 /patch_qwen3_tts_runtime.py
|
||||
fi
|
||||
|
||||
# --- Browser-UI im Container starten ---
|
||||
if [ -f /voice_clone_ui.py ]; then
|
||||
python3 /voice_clone_ui.py \
|
||||
--host 0.0.0.0 \
|
||||
--port "$UI_PORT" \
|
||||
--api-base "http://localhost:${PORT}" \
|
||||
--clone-api-base "http://qwen3-tts-clone:8093" \
|
||||
--model "$MODEL" &
|
||||
UI_PID=$!
|
||||
echo "[ui] Voice-Cloning-UI laeuft auf Port ${UI_PORT}"
|
||||
fi
|
||||
|
||||
# --- WebSocket-Logging-Proxy im Container starten ---
|
||||
if [ -f /ws_log_proxy.py ]; then
|
||||
python3 /ws_log_proxy.py \
|
||||
--host 0.0.0.0 \
|
||||
--port "$WS_LOG_PORT" \
|
||||
--upstream "ws://localhost:${PORT}/v1/audio/speech/stream" &
|
||||
WS_LOG_PID=$!
|
||||
echo "[ws-log] Proxy laeuft auf Port ${WS_LOG_PORT}"
|
||||
fi
|
||||
|
||||
# --- Server im Hintergrund starten ---
|
||||
vllm-omni serve "$MODEL" \
|
||||
@@ -38,4 +67,5 @@ SERVER_PID=$!
|
||||
) &
|
||||
|
||||
# --- Auf den Server-Prozess warten (PID 1 Signalweiterleitung) ---
|
||||
trap 'kill "$SERVER_PID" "${UI_PID:-}" "${WS_LOG_PID:-}" 2>/dev/null || true' TERM INT
|
||||
wait "$SERVER_PID"
|
||||
|
||||
14
entrypoint_clone.sh
Executable file
14
entrypoint_clone.sh
Executable file
@@ -0,0 +1,14 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
MODEL="${QWEN3_TTS_CLONE_MODEL:-}"
|
||||
PORT="${QWEN3_TTS_CLONE_PORT:-8093}"
|
||||
|
||||
if [ -z "$MODEL" ]; then
|
||||
echo "[clone] QWEN3_TTS_CLONE_MODEL ist nicht gesetzt." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
echo "[clone] starte Clone-Modell: ${MODEL} auf Port ${PORT}"
|
||||
|
||||
exec vllm-omni serve "$MODEL" --omni --host 0.0.0.0 --port "$PORT" --deploy-config /deploy/qwen3_tts.yaml --gpu-memory-utilization 0.10 --trust-remote-code
|
||||
134
patch_qwen3_tts_runtime.py
Normal file
134
patch_qwen3_tts_runtime.py
Normal file
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Runtime patches for the vllm-omni Qwen3-TTS container.
|
||||
|
||||
- Propagate WebSocket session.config.seed into every generated speech segment.
|
||||
- Log WebSocket config and segment generation on the real 8091 endpoint.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
STREAM = Path('/usr/local/lib/python3.12/dist-packages/vllm_omni/entrypoints/openai/serving_speech_stream.py')
|
||||
AUDIO = Path('/usr/local/lib/python3.12/dist-packages/vllm_omni/entrypoints/openai/protocol/audio.py')
|
||||
|
||||
|
||||
def replace_once(text: str, old: str, new: str, label: str) -> str:
|
||||
if new in text:
|
||||
return text
|
||||
if old not in text:
|
||||
raise RuntimeError(f'Patch anchor not found: {label}')
|
||||
return text.replace(old, new, 1)
|
||||
|
||||
|
||||
def patch_seed_field() -> None:
|
||||
"""Add an optional `seed` field to StreamingSpeechSessionConfig.
|
||||
|
||||
The streaming WebSocket config model in protocol/audio.py ships without a
|
||||
`seed` attribute, but the stream server reads `config.seed` (see seed
|
||||
propagation patch below). Without this field every session.config fails with
|
||||
AttributeError: 'StreamingSpeechSessionConfig' object has no attribute 'seed'.
|
||||
"""
|
||||
text = AUDIO.read_text()
|
||||
# Anchor on this class's unique docstring + first field: the task_type line
|
||||
# is not unique (it also appears in SpeechBatchItem / BatchSpeechRequest).
|
||||
new = replace_once(
|
||||
text,
|
||||
' """Configuration sent as the first WebSocket message for streaming TTS."""\n\n'
|
||||
' model: str | None = None\n',
|
||||
' """Configuration sent as the first WebSocket message for streaming TTS."""\n\n'
|
||||
' seed: int | None = None\n'
|
||||
' model: str | None = None\n',
|
||||
'StreamingSpeechSessionConfig.seed field',
|
||||
)
|
||||
if new != text:
|
||||
AUDIO.write_text(new)
|
||||
print('[runtime-patch] StreamingSpeechSessionConfig.seed field added', flush=True)
|
||||
else:
|
||||
print('[runtime-patch] StreamingSpeechSessionConfig.seed field already present', flush=True)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
patch_seed_field()
|
||||
|
||||
text = STREAM.read_text()
|
||||
original = text
|
||||
|
||||
text = replace_once(
|
||||
text,
|
||||
' if config.model and hasattr(self._speech_service, "_check_model"):\n',
|
||||
' logger.info(\n'
|
||||
' "Streaming speech config: model=%s voice=%s task_type=%s language=%s seed=%s split=%s stream_audio=%s instructions=%r speaker_embedding_dim=%s",\n'
|
||||
' config.model,\n'
|
||||
' config.voice,\n'
|
||||
' config.task_type,\n'
|
||||
' config.language,\n'
|
||||
' config.seed,\n'
|
||||
' config.split_granularity,\n'
|
||||
' config.stream_audio,\n'
|
||||
' config.instructions,\n'
|
||||
' len(config.speaker_embedding) if config.speaker_embedding is not None else None,\n'
|
||||
' )\n\n'
|
||||
' if config.model and hasattr(self._speech_service, "_check_model"):\n',
|
||||
'config logging',
|
||||
)
|
||||
|
||||
text = replace_once(
|
||||
text,
|
||||
' stream=config.stream_audio,\n )\n',
|
||||
' stream=config.stream_audio,\n seed=config.seed,\n )\n'
|
||||
' logger.info(\n'
|
||||
' "Streaming speech segment: index=%s chars=%s voice=%s task_type=%s language=%s seed=%s speaker_embedding_dim=%s",\n'
|
||||
' sentence_index,\n'
|
||||
' len(sentence_text),\n'
|
||||
' config.voice,\n'
|
||||
' config.task_type,\n'
|
||||
' config.language,\n'
|
||||
' config.seed,\n'
|
||||
' len(config.speaker_embedding) if config.speaker_embedding is not None else None,\n'
|
||||
' )\n',
|
||||
'seed propagation and segment logging',
|
||||
)
|
||||
|
||||
# --- Silence padding (PCM streaming only) ---
|
||||
# Lead-in silence at session start gives the client time to set up smooth
|
||||
# playback before real audio (mitigates start-of-stream stutter). Trailing
|
||||
# silence avoids the abrupt cut-off at the end. Both in ms via env vars.
|
||||
text = replace_once(
|
||||
text,
|
||||
'_PCM_SAMPLE_RATE = 24000\n',
|
||||
'_PCM_SAMPLE_RATE = 24000\n'
|
||||
'import os as _os\n'
|
||||
'_LEADIN_SILENCE_MS = int(_os.environ.get("QWEN3_TTS_LEADIN_SILENCE_MS", "250"))\n'
|
||||
'_TRAILING_SILENCE_MS = int(_os.environ.get("QWEN3_TTS_TRAILING_SILENCE_MS", "300"))\n'
|
||||
'_LEADIN_SILENCE_BYTES = b"\\x00" * (int(_PCM_SAMPLE_RATE * _LEADIN_SILENCE_MS / 1000) * 2)\n'
|
||||
'_TRAILING_SILENCE_BYTES = b"\\x00" * (int(_PCM_SAMPLE_RATE * _TRAILING_SILENCE_MS / 1000) * 2)\n',
|
||||
'silence padding constants',
|
||||
)
|
||||
|
||||
text = replace_once(
|
||||
text,
|
||||
' await websocket.send_json(start_payload)\n',
|
||||
' await websocket.send_json(start_payload)\n'
|
||||
' if config.stream_audio and response_format == "pcm" and sentence_index == 0 and _LEADIN_SILENCE_BYTES:\n'
|
||||
' await websocket.send_bytes(_LEADIN_SILENCE_BYTES)\n',
|
||||
'lead-in silence',
|
||||
)
|
||||
|
||||
text = replace_once(
|
||||
text,
|
||||
' # Send session.done\n'
|
||||
' await websocket.send_json(\n',
|
||||
' if config.stream_audio and config.response_format == "pcm" and _TRAILING_SILENCE_BYTES:\n'
|
||||
' await websocket.send_bytes(_TRAILING_SILENCE_BYTES)\n\n'
|
||||
' # Send session.done\n'
|
||||
' await websocket.send_json(\n',
|
||||
'trailing silence',
|
||||
)
|
||||
|
||||
if text != original:
|
||||
STREAM.write_text(text)
|
||||
print('[runtime-patch] WebSocket seed propagation/logging + silence padding applied', flush=True)
|
||||
else:
|
||||
print('[runtime-patch] WebSocket seed propagation/logging already present', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
67
start.sh
67
start.sh
@@ -1,69 +1,4 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
HEALTH_URL="http://localhost:8091/health"
|
||||
MAX_WAIT=180
|
||||
|
||||
strip_ansi() { sed 's/\x1b\[[0-9;]*[mK]//g'; }
|
||||
|
||||
t0=$SECONDS
|
||||
|
||||
# --- Container starten ---
|
||||
printf "Container startet..."
|
||||
docker compose up -d &>/tmp/qwen3-tts-compose.log
|
||||
printf " [%ds]\n\n" $(( SECONDS - t0 ))
|
||||
|
||||
# --- Log-Events im Hintergrund parsen und ausgeben ---
|
||||
docker logs -f qwen3-tts 2>&1 | strip_ansi | while IFS= read -r line; do
|
||||
case "$line" in
|
||||
*"Stage 0 engine launch started"*)
|
||||
printf " Stage 0 Talker LLM wird geladen\n" ;;
|
||||
*"Initializing a V1 LLM engine"*)
|
||||
printf " Stage 0 Engine initialisiert\n" ;;
|
||||
*"Loading weights"*)
|
||||
printf " Stage 0 Modell-Gewichte werden geladen...\n" ;;
|
||||
*"Loading model weights took"*)
|
||||
took=$(echo "$line" | grep -oP '\d+\.\d+s' | head -1)
|
||||
printf " Stage 0 Gewichte geladen (%s)\n" "$took" ;;
|
||||
*"Stage 1 engine launch started"*)
|
||||
printf " Stage 1 Code2Wav Decoder wird geladen\n" ;;
|
||||
*"Graph capturing finished"*)
|
||||
printf " CUDA Graphen kompiliert\n" ;;
|
||||
*"Application startup complete"*)
|
||||
printf " Server FastAPI bereit\n" ;;
|
||||
*"Uvicorn running on"*)
|
||||
printf " Server HTTP läuft auf Port 8091\n" ;;
|
||||
*"warmup] Server gesund"*)
|
||||
printf " Warmup Aufwärm-Request läuft (kompiliert/lädt Graphen)...\n" ;;
|
||||
*"warmup] fertig"*)
|
||||
took=$(echo "$line" | grep -oP '\d+s' | head -1)
|
||||
printf " Warmup Server ist heiß (%s)\n" "$took" ;;
|
||||
*"ERROR"*|*"ValueError"*)
|
||||
msg=$(echo "$line" | grep -oP '(ERROR|ValueError).*' | head -1)
|
||||
printf " [!] %s\n" "${msg:0:100}" ;;
|
||||
esac
|
||||
done &
|
||||
LOG_PID=$!
|
||||
|
||||
# --- Health-Check-Schleife ---
|
||||
while (( SECONDS - t0 < MAX_WAIT )); do
|
||||
if curl -sf "$HEALTH_URL" >/dev/null 2>&1; then
|
||||
kill "$LOG_PID" 2>/dev/null
|
||||
wait "$LOG_PID" 2>/dev/null
|
||||
printf "\n✓ Bereit: http://localhost:8091 [%ds]\n" $(( SECONDS - t0 ))
|
||||
exit 0
|
||||
fi
|
||||
# Abbruch wenn Container schon gestoppt ist
|
||||
if ! docker ps --filter "name=qwen3-tts" --filter "status=running" -q | grep -q .; then
|
||||
kill "$LOG_PID" 2>/dev/null
|
||||
printf "\n✗ Container abgestürzt. Logs:\n"
|
||||
docker logs --tail=20 qwen3-tts 2>&1 | strip_ansi | grep -E "ERROR|ValueError|Exception" | head -5
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
kill "$LOG_PID" 2>/dev/null
|
||||
printf "\n✗ Timeout nach %ds. Logs: docker logs qwen3-tts\n" "$MAX_WAIT" >&2
|
||||
exit 1
|
||||
exec ./start_tts.sh "$@"
|
||||
|
||||
43
start_clone.sh
Executable file
43
start_clone.sh
Executable file
@@ -0,0 +1,43 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
HEALTH_URL="http://localhost:8093/health"
|
||||
MAX_WAIT="${MAX_WAIT:-180}"
|
||||
MODEL="${QWEN3_TTS_CLONE_MODEL:-Qwen/Qwen3-TTS-12Hz-1.7B-Base}"
|
||||
|
||||
t0=$SECONDS
|
||||
printf "TTS-Container stoppt..."
|
||||
docker compose stop qwen3-tts >/dev/null 2>&1 || true
|
||||
printf " [%ds]
|
||||
" $(( SECONDS - t0 ))
|
||||
|
||||
printf "Clone-Container startet..."
|
||||
QWEN3_TTS_CLONE_MODEL="$MODEL" docker compose --profile clone up -d qwen3-tts-clone >/tmp/qwen3-tts-clone-compose.log 2>&1
|
||||
printf " [%ds]
|
||||
" $(( SECONDS - t0 ))
|
||||
printf " Modell %s
|
||||
" "$MODEL"
|
||||
printf " API http://localhost:8093
|
||||
|
||||
"
|
||||
|
||||
while (( SECONDS - t0 < MAX_WAIT )); do
|
||||
if curl -sf "$HEALTH_URL" >/dev/null 2>&1; then
|
||||
printf "✓ Clone bereit: http://localhost:8093 [%ds]
|
||||
" $(( SECONDS - t0 ))
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! docker ps --filter "name=qwen3-tts-clone" --filter "status=running" -q | grep -q .; then
|
||||
printf "✗ Clone-Container abgestürzt. Logs:
|
||||
" >&2
|
||||
docker logs --tail=40 qwen3-tts-clone 2>&1 >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
printf "✗ Timeout nach %ss. Logs: docker logs qwen3-tts-clone
|
||||
" "$MAX_WAIT" >&2
|
||||
exit 1
|
||||
5
start_custom_tts.sh
Executable file
5
start_custom_tts.sh
Executable file
@@ -0,0 +1,5 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
export QWEN3_TTS_MODEL="${QWEN3_TTS_MODEL:-Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice}"
|
||||
exec ./start_tts.sh
|
||||
5
start_dialog_tts.sh
Executable file
5
start_dialog_tts.sh
Executable file
@@ -0,0 +1,5 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
export QWEN3_TTS_MODEL="${QWEN3_TTS_MODEL:-Qwen/Qwen3-TTS-12Hz-1.7B-Base}"
|
||||
exec ./start_tts.sh
|
||||
77
start_tts.sh
Executable file
77
start_tts.sh
Executable file
@@ -0,0 +1,77 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
HEALTH_URL="http://localhost:8091/health"
|
||||
MAX_WAIT=180
|
||||
MODEL="${QWEN3_TTS_MODEL:-Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice}"
|
||||
|
||||
strip_ansi() { sed 's/\x1b\[[0-9;]*[mK]//g'; }
|
||||
|
||||
t0=$SECONDS
|
||||
|
||||
# --- Gegenseitigen Service stoppen ---
|
||||
printf "Clone-Container stoppt..."
|
||||
docker compose --profile clone stop qwen3-tts-clone >/dev/null 2>&1 || true
|
||||
printf " [%ds]
|
||||
" $(( SECONDS - t0 ))
|
||||
|
||||
# --- Container starten ---
|
||||
printf "TTS-Container startet..."
|
||||
QWEN3_TTS_MODEL="$MODEL" docker compose up -d qwen3-tts &>/tmp/qwen3-tts-compose.log
|
||||
printf " [%ds]\n" $(( SECONDS - t0 ))
|
||||
printf " Modell %s\n\n" "$MODEL"
|
||||
|
||||
# --- Log-Events im Hintergrund parsen und ausgeben ---
|
||||
docker logs -f qwen3-tts 2>&1 | strip_ansi | while IFS= read -r line; do
|
||||
case "$line" in
|
||||
*"Stage 0 engine launch started"*)
|
||||
printf " Stage 0 Talker LLM wird geladen\n" ;;
|
||||
*"Initializing a V1 LLM engine"*)
|
||||
printf " Stage 0 Engine initialisiert\n" ;;
|
||||
*"Loading weights"*)
|
||||
printf " Stage 0 Modell-Gewichte werden geladen...\n" ;;
|
||||
*"Loading model weights took"*)
|
||||
took=$(echo "$line" | grep -oP '\d+\.\d+s' | head -1)
|
||||
printf " Stage 0 Gewichte geladen (%s)\n" "$took" ;;
|
||||
*"Stage 1 engine launch started"*)
|
||||
printf " Stage 1 Code2Wav Decoder wird geladen\n" ;;
|
||||
*"Graph capturing finished"*)
|
||||
printf " CUDA Graphen kompiliert\n" ;;
|
||||
*"Application startup complete"*)
|
||||
printf " Server FastAPI bereit\n" ;;
|
||||
*"Uvicorn running on"*)
|
||||
printf " Server HTTP läuft auf Port 8091\n" ;;
|
||||
*"warmup] Server gesund"*)
|
||||
printf " Warmup Aufwärm-Request läuft (kompiliert/lädt Graphen)...\n" ;;
|
||||
*"warmup] fertig"*)
|
||||
took=$(echo "$line" | grep -oP '\d+s' | head -1)
|
||||
printf " Warmup Server ist heiß (%s)\n" "$took" ;;
|
||||
*"ERROR"*|*"ValueError"*)
|
||||
msg=$(echo "$line" | grep -oP '(ERROR|ValueError).*' | head -1)
|
||||
printf " [!] %s\n" "${msg:0:100}" ;;
|
||||
esac
|
||||
done &
|
||||
LOG_PID=$!
|
||||
|
||||
# --- Health-Check-Schleife ---
|
||||
while (( SECONDS - t0 < MAX_WAIT )); do
|
||||
if curl -sf "$HEALTH_URL" >/dev/null 2>&1; then
|
||||
kill "$LOG_PID" 2>/dev/null
|
||||
wait "$LOG_PID" 2>/dev/null || true
|
||||
printf "\n✓ Bereit: API http://localhost:8091 UI http://localhost:8092 WS-Log ws://localhost:8094/v1/audio/speech/stream [%ds]\n" $(( SECONDS - t0 ))
|
||||
exit 0
|
||||
fi
|
||||
# Abbruch wenn Container schon gestoppt ist
|
||||
if ! docker ps --filter "name=qwen3-tts" --filter "status=running" -q | grep -q .; then
|
||||
kill "$LOG_PID" 2>/dev/null
|
||||
printf "\n✗ Container abgestürzt. Logs:\n"
|
||||
docker logs --tail=20 qwen3-tts 2>&1 | strip_ansi | grep -E "ERROR|ValueError|Exception" | head -5
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
kill "$LOG_PID" 2>/dev/null
|
||||
printf "\n✗ Timeout nach %ds. Logs: docker logs qwen3-tts\n" "$MAX_WAIT" >&2
|
||||
exit 1
|
||||
8
stop_clone.sh
Executable file
8
stop_clone.sh
Executable file
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
docker compose --profile clone stop qwen3-tts-clone >/dev/null 2>&1 || true
|
||||
docker compose --profile clone rm -f qwen3-tts-clone >/dev/null 2>&1 || true
|
||||
printf "✓ Clone-Service gestoppt
|
||||
"
|
||||
8
stop_tts.sh
Executable file
8
stop_tts.sh
Executable file
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
docker compose stop qwen3-tts >/dev/null 2>&1 || true
|
||||
docker compose rm -f qwen3-tts >/dev/null 2>&1 || true
|
||||
printf "✓ TTS/UI gestoppt
|
||||
"
|
||||
40
test_ws.py
40
test_ws.py
@@ -6,6 +6,7 @@ import time
|
||||
import wave
|
||||
import struct
|
||||
import asyncio
|
||||
import re
|
||||
|
||||
try:
|
||||
import websockets
|
||||
@@ -18,12 +19,30 @@ except ImportError:
|
||||
WS_URL = "ws://localhost:8091/v1/audio/speech/stream"
|
||||
OUTPUT_FILE = "test_stream_output.wav"
|
||||
SAMPLE_RATE = 24000
|
||||
WORD_RE = re.compile(r"\S+\s*")
|
||||
|
||||
|
||||
async def stream_tts(text: str, voice: str = "Ryan", language: str = "German"):
|
||||
def iter_words(text: str):
|
||||
"""Yield complete words, preserving trailing whitespace.
|
||||
|
||||
The WebSocket client must not forward raw LLM tokens such as partial
|
||||
subwords. It buffers to words and lets the server detect sentence ends.
|
||||
"""
|
||||
for match in WORD_RE.finditer(text):
|
||||
yield match.group(0)
|
||||
|
||||
|
||||
async def stream_tts(
|
||||
text: str,
|
||||
voice: str = "Ryan",
|
||||
language: str = "German",
|
||||
instructions: str = "Eine warme, ruhige deutsche Stimme, klar artikuliert, freundlich und natuerlich.",
|
||||
seed: int = 42,
|
||||
):
|
||||
print(f"[→] WebSocket connect: {WS_URL}")
|
||||
print(f" text: {text!r}")
|
||||
print(f" voice: {voice}, language: {language}")
|
||||
print(f" voice: {voice}, language: {language}, seed: {seed}")
|
||||
print(f" instruct: {instructions!r}")
|
||||
|
||||
pcm_chunks = []
|
||||
total_bytes = 0
|
||||
@@ -32,20 +51,27 @@ async def stream_tts(text: str, voice: str = "Ryan", language: str = "German"):
|
||||
|
||||
async with websockets.connect(WS_URL) as ws:
|
||||
# Send session config
|
||||
await ws.send(json.dumps({
|
||||
config = {
|
||||
"type": "session.config",
|
||||
"model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
|
||||
"task_type": "CustomVoice",
|
||||
"task_type": "VoiceDesign",
|
||||
"voice": voice,
|
||||
"language": language,
|
||||
"instructions": instructions,
|
||||
"seed": seed,
|
||||
"response_format": "pcm",
|
||||
"stream_audio": True,
|
||||
}))
|
||||
"split_granularity": "sentence",
|
||||
}
|
||||
print("[cfg]", json.dumps(config, ensure_ascii=False, sort_keys=True), flush=True)
|
||||
await ws.send(json.dumps(config))
|
||||
|
||||
# Send text
|
||||
# Send word-by-word. Do not split into sentences client-side; the
|
||||
# server still decides when a complete sentence is ready for audio.
|
||||
for word in iter_words(text):
|
||||
await ws.send(json.dumps({
|
||||
"type": "input.text",
|
||||
"text": text,
|
||||
"text": word,
|
||||
}))
|
||||
await ws.send(json.dumps({"type": "input.done"}))
|
||||
|
||||
|
||||
900
ui/voice-cloning.html
Normal file
900
ui/voice-cloning.html
Normal file
@@ -0,0 +1,900 @@
|
||||
<!doctype html>
|
||||
<html lang="de">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Qwen3-TTS Voice Cloning</title>
|
||||
<style>
|
||||
:root {
|
||||
color-scheme: light;
|
||||
--bg: #f7f8fa;
|
||||
--panel: #ffffff;
|
||||
--panel-soft: #eef2f6;
|
||||
--text: #17202a;
|
||||
--muted: #5d6a78;
|
||||
--line: #d9e0e7;
|
||||
--accent: #176b5d;
|
||||
--accent-strong: #0f5147;
|
||||
--danger: #a63a3a;
|
||||
--focus: #3478f6;
|
||||
}
|
||||
|
||||
* { box-sizing: border-box; }
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
min-height: 100vh;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
||||
letter-spacing: 0;
|
||||
}
|
||||
|
||||
main {
|
||||
width: min(1160px, calc(100vw - 32px));
|
||||
margin: 0 auto;
|
||||
padding: 28px 0 36px;
|
||||
}
|
||||
|
||||
header {
|
||||
display: flex;
|
||||
align-items: end;
|
||||
justify-content: space-between;
|
||||
gap: 18px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
h1 {
|
||||
margin: 0;
|
||||
font-size: clamp(1.7rem, 3vw, 2.45rem);
|
||||
line-height: 1.05;
|
||||
font-weight: 760;
|
||||
}
|
||||
|
||||
.status {
|
||||
min-width: 170px;
|
||||
padding: 9px 12px;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
background: var(--panel);
|
||||
color: var(--muted);
|
||||
font-size: 0.92rem;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.status[data-state="ok"] {
|
||||
color: var(--accent-strong);
|
||||
border-color: #9bc6bd;
|
||||
background: #edf7f4;
|
||||
}
|
||||
|
||||
.status[data-state="error"] {
|
||||
color: var(--danger);
|
||||
border-color: #e2b6b6;
|
||||
background: #fff3f3;
|
||||
}
|
||||
|
||||
form {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) 380px;
|
||||
gap: 18px;
|
||||
}
|
||||
|
||||
section, aside { min-width: 0; }
|
||||
|
||||
.panel {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
background: var(--panel);
|
||||
}
|
||||
|
||||
.panel + .panel { margin-top: 18px; }
|
||||
|
||||
.panel-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
padding: 14px 16px;
|
||||
border-bottom: 1px solid var(--line);
|
||||
background: var(--panel-soft);
|
||||
border-radius: 8px 8px 0 0;
|
||||
}
|
||||
|
||||
h2 {
|
||||
margin: 0;
|
||||
font-size: 1rem;
|
||||
line-height: 1.2;
|
||||
}
|
||||
|
||||
.panel-body { padding: 16px; }
|
||||
|
||||
label {
|
||||
display: block;
|
||||
margin-bottom: 7px;
|
||||
color: var(--muted);
|
||||
font-size: 0.86rem;
|
||||
font-weight: 650;
|
||||
}
|
||||
|
||||
input, textarea, select {
|
||||
width: 100%;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 7px;
|
||||
background: #fff;
|
||||
color: var(--text);
|
||||
font: inherit;
|
||||
}
|
||||
|
||||
input, select {
|
||||
height: 40px;
|
||||
padding: 0 10px;
|
||||
}
|
||||
|
||||
textarea {
|
||||
min-height: 118px;
|
||||
resize: vertical;
|
||||
padding: 10px;
|
||||
line-height: 1.45;
|
||||
}
|
||||
|
||||
input:focus, textarea:focus, select:focus, button:focus-visible {
|
||||
outline: 2px solid var(--focus);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
|
||||
.grid-2 {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 14px;
|
||||
}
|
||||
|
||||
.field + .field, .grid-2 + .field, .field + .grid-2, .field + .actions {
|
||||
margin-top: 14px;
|
||||
}
|
||||
|
||||
.file-row {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) auto;
|
||||
gap: 10px;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.file-meta {
|
||||
color: var(--muted);
|
||||
font-size: 0.84rem;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.actions {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 10px;
|
||||
align-items: center;
|
||||
justify-content: flex-end;
|
||||
}
|
||||
|
||||
button {
|
||||
min-height: 40px;
|
||||
border: 1px solid transparent;
|
||||
border-radius: 7px;
|
||||
padding: 0 14px;
|
||||
color: #fff;
|
||||
background: var(--accent);
|
||||
font: inherit;
|
||||
font-weight: 720;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
button:hover { background: var(--accent-strong); }
|
||||
|
||||
button.secondary {
|
||||
color: var(--text);
|
||||
border-color: var(--line);
|
||||
background: #fff;
|
||||
}
|
||||
|
||||
button.secondary:hover { background: #f3f6f8; }
|
||||
|
||||
button.danger {
|
||||
color: var(--danger);
|
||||
border-color: #e2b6b6;
|
||||
background: #fff;
|
||||
}
|
||||
|
||||
button.danger:hover { background: #fff3f3; }
|
||||
|
||||
button:disabled { cursor: wait; opacity: 0.65; }
|
||||
|
||||
audio {
|
||||
width: 100%;
|
||||
margin-top: 14px;
|
||||
}
|
||||
|
||||
.download {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
min-height: 40px;
|
||||
margin-top: 12px;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 7px;
|
||||
padding: 0 12px;
|
||||
color: var(--text);
|
||||
background: #fff;
|
||||
font-weight: 700;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.download[hidden] { display: none; }
|
||||
|
||||
pre {
|
||||
min-height: 160px;
|
||||
max-height: 360px;
|
||||
overflow: auto;
|
||||
margin: 0;
|
||||
padding: 12px;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 7px;
|
||||
background: #101820;
|
||||
color: #d7e1ea;
|
||||
font: 0.82rem/1.45 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.hidden { display: none; }
|
||||
|
||||
@media (max-width: 880px) {
|
||||
main {
|
||||
width: min(100vw - 20px, 720px);
|
||||
padding-top: 16px;
|
||||
}
|
||||
|
||||
header, form, .grid-2, .file-row {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
header {
|
||||
display: grid;
|
||||
align-items: stretch;
|
||||
}
|
||||
|
||||
.status { text-align: left; }
|
||||
.actions { justify-content: stretch; }
|
||||
.actions button { flex: 1 1 180px; }
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<header>
|
||||
<h1>Qwen3-TTS Voice Cloning</h1>
|
||||
<div id="status" class="status">Bereit</div>
|
||||
</header>
|
||||
|
||||
<form id="cloneForm">
|
||||
<section>
|
||||
<div class="panel">
|
||||
<div class="panel-header"><h2>Quelle</h2></div>
|
||||
<div class="panel-body">
|
||||
<div class="field">
|
||||
<label for="sourceMode">Synthesequelle</label>
|
||||
<select id="sourceMode" name="sourceMode">
|
||||
<option value="saved" selected>Gespeicherte Stimme</option>
|
||||
<option value="design">Voice Design</option>
|
||||
<option value="vector">Speaker Vector</option>
|
||||
<option value="adhoc">Ad-hoc Referenz</option>
|
||||
</select>
|
||||
</div>
|
||||
<div id="savedVoiceBox">
|
||||
<div class="grid-2">
|
||||
<div class="field">
|
||||
<label for="savedVoice">Stimme</label>
|
||||
<select id="savedVoice" name="savedVoice"></select>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="voiceTaskType">Task</label>
|
||||
<select id="voiceTaskType" name="voiceTaskType">
|
||||
<option value="Base" selected>Base</option>
|
||||
<option value="CustomVoice">CustomVoice</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="actions field">
|
||||
<button id="refreshVoicesButton" class="secondary" type="button">Stimmen laden</button>
|
||||
<button id="deleteVoiceButton" class="danger" type="button">Stimme loeschen</button>
|
||||
</div>
|
||||
</div>
|
||||
<div id="voiceDesignBox" class="hidden">
|
||||
<div class="grid-2">
|
||||
<div class="field">
|
||||
<label for="designVoice">Ausgangsstimme</label>
|
||||
<select id="designVoice" name="designVoice"></select>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="designSeed">Seed</label>
|
||||
<input id="designSeed" name="designSeed" type="number" min="0" step="1" value="42">
|
||||
</div>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="designDescription">Beschreibung</label>
|
||||
<textarea id="designDescription" name="designDescription">Eine warme, ruhige deutsche Stimme, klar artikuliert, freundlich und natuerlich.</textarea>
|
||||
</div>
|
||||
<div class="actions field">
|
||||
<button id="savePreviewButton" class="secondary" type="button" disabled>Vorschau als Stimme speichern</button>
|
||||
</div>
|
||||
</div>
|
||||
<div id="vectorBox" class="hidden">
|
||||
<div class="grid-2">
|
||||
<div class="field">
|
||||
<label for="vectorSelect">Gespeicherter Vector</label>
|
||||
<select id="vectorSelect" name="vectorSelect"></select>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="vectorName">Vector-Name</label>
|
||||
<input id="vectorName" name="vectorName" value="stimme_vector_01">
|
||||
</div>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="vectorDescription">Beschreibung</label>
|
||||
<input id="vectorDescription" name="vectorDescription" value="stabiler 2048er Speaker-Vector">
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="vectorJson">Vector JSON</label>
|
||||
<textarea id="vectorJson" name="vectorJson" placeholder="[0.01, -0.02, ...]"></textarea>
|
||||
</div>
|
||||
<div class="actions field">
|
||||
<button id="refreshVectorsButton" class="secondary" type="button">Vectoren laden</button>
|
||||
<button id="saveVectorButton" type="button">Vector speichern</button>
|
||||
<button id="deleteVectorButton" class="danger" type="button">Vector loeschen</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
<div class="panel-header"><h2>Referenz speichern</h2></div>
|
||||
<div class="panel-body">
|
||||
<div class="grid-2">
|
||||
<div class="field">
|
||||
<label for="voiceName">Name</label>
|
||||
<input id="voiceName" name="voiceName" value="meine_stimme">
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="consent">Consent</label>
|
||||
<input id="consent" name="consent" value="local-consent">
|
||||
</div>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="refAudio">Referenzaudio</label>
|
||||
<div class="file-row">
|
||||
<input id="refAudio" name="refAudio" type="file" accept="audio/*">
|
||||
<span id="fileMeta" class="file-meta">Keine Datei</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="refText">Transkript der Referenz</label>
|
||||
<textarea id="refText" name="refText">Dies ist eine kurze Referenzaufnahme fuer das Klonen meiner Stimme.</textarea>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="speakerDescription">Beschreibung</label>
|
||||
<input id="speakerDescription" name="speakerDescription" value="ruhige deutsche Stimme">
|
||||
</div>
|
||||
<div class="actions">
|
||||
<button id="saveVoiceButton" type="button">Stimme speichern</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
<div class="panel-header"><h2>Testausgabe</h2></div>
|
||||
<div class="panel-body">
|
||||
<div class="field">
|
||||
<label for="inputText">Zieltext</label>
|
||||
<textarea id="inputText" name="inputText" required>Hallo, dies ist ein Test der geklonten Stimme mit Qwen3-TTS.</textarea>
|
||||
</div>
|
||||
<div class="actions">
|
||||
<button id="resetButton" class="secondary" type="button">Zuruecksetzen</button>
|
||||
<button id="submitButton" type="submit">Test erzeugen</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<aside>
|
||||
<div class="panel">
|
||||
<div class="panel-header"><h2>Parameter</h2></div>
|
||||
<div class="panel-body">
|
||||
<div class="field">
|
||||
<label for="speechEndpoint">Speech Endpoint</label>
|
||||
<input id="speechEndpoint" name="speechEndpoint" value="/api/v1/audio/speech" required>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="voicesEndpoint">Voices Endpoint</label>
|
||||
<input id="voicesEndpoint" name="voicesEndpoint" value="/api/v1/audio/voices" required>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="vectorsEndpoint">Vector Endpoint</label>
|
||||
<input id="vectorsEndpoint" name="vectorsEndpoint" value="/api/voice-vectors" required>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="model">Modell</label>
|
||||
<input id="model" name="model" value="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice" required>
|
||||
</div>
|
||||
<div class="grid-2">
|
||||
<div class="field">
|
||||
<label for="language">Sprache</label>
|
||||
<select id="language" name="language">
|
||||
<option value="German" selected>German</option>
|
||||
<option value="English">English</option>
|
||||
<option value="Auto">Auto</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="speed">Tempo</label>
|
||||
<input id="speed" name="speed" type="number" min="0.5" max="2" step="0.05" value="1">
|
||||
</div>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="audioEncoding">Ad-hoc Audiofeld</label>
|
||||
<select id="audioEncoding" name="audioEncoding">
|
||||
<option value="data_url" selected>Data URL</option>
|
||||
<option value="base64">Base64</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="instructions">Stil</label>
|
||||
<textarea id="instructions" name="instructions">Sprich natuerlich, ruhig und deutlich.</textarea>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
<div class="panel-header"><h2>Ergebnis</h2></div>
|
||||
<div class="panel-body">
|
||||
<audio id="player" controls></audio>
|
||||
<a id="download" class="download" href="#" download="voice-clone-output.wav" hidden>WAV herunterladen</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
<div class="panel-header"><h2>Log</h2></div>
|
||||
<div class="panel-body"><pre id="log"></pre></div>
|
||||
</div>
|
||||
</aside>
|
||||
</form>
|
||||
</main>
|
||||
|
||||
<script>
|
||||
const form = document.querySelector("#cloneForm");
|
||||
const statusBox = document.querySelector("#status");
|
||||
const submitButton = document.querySelector("#submitButton");
|
||||
const saveVoiceButton = document.querySelector("#saveVoiceButton");
|
||||
const refreshVoicesButton = document.querySelector("#refreshVoicesButton");
|
||||
const deleteVoiceButton = document.querySelector("#deleteVoiceButton");
|
||||
const savePreviewButton = document.querySelector("#savePreviewButton");
|
||||
const refreshVectorsButton = document.querySelector("#refreshVectorsButton");
|
||||
const saveVectorButton = document.querySelector("#saveVectorButton");
|
||||
const deleteVectorButton = document.querySelector("#deleteVectorButton");
|
||||
const resetButton = document.querySelector("#resetButton");
|
||||
const refAudio = document.querySelector("#refAudio");
|
||||
const fileMeta = document.querySelector("#fileMeta");
|
||||
const sourceMode = document.querySelector("#sourceMode");
|
||||
const savedVoiceBox = document.querySelector("#savedVoiceBox");
|
||||
const voiceDesignBox = document.querySelector("#voiceDesignBox");
|
||||
const vectorBox = document.querySelector("#vectorBox");
|
||||
const savedVoice = document.querySelector("#savedVoice");
|
||||
const designVoice = document.querySelector("#designVoice");
|
||||
const vectorSelect = document.querySelector("#vectorSelect");
|
||||
const player = document.querySelector("#player");
|
||||
const download = document.querySelector("#download");
|
||||
const logBox = document.querySelector("#log");
|
||||
let currentObjectUrl = "";
|
||||
let lastGeneratedBlob = null;
|
||||
let lastGeneratedText = "";
|
||||
|
||||
function setStatus(text, state = "") {
|
||||
statusBox.textContent = text;
|
||||
statusBox.dataset.state = state;
|
||||
}
|
||||
|
||||
function log(line) {
|
||||
const time = new Date().toLocaleTimeString();
|
||||
logBox.textContent += `[${time}] ${line}\n`;
|
||||
logBox.scrollTop = logBox.scrollHeight;
|
||||
}
|
||||
|
||||
function resetResult() {
|
||||
if (currentObjectUrl) {
|
||||
URL.revokeObjectURL(currentObjectUrl);
|
||||
currentObjectUrl = "";
|
||||
}
|
||||
player.removeAttribute("src");
|
||||
download.hidden = true;
|
||||
download.removeAttribute("href");
|
||||
}
|
||||
|
||||
function fileToDataUrl(file) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => resolve(String(reader.result));
|
||||
reader.onerror = () => reject(reader.error || new Error("Datei konnte nicht gelesen werden."));
|
||||
reader.readAsDataURL(file);
|
||||
});
|
||||
}
|
||||
|
||||
function setBusy(isBusy) {
|
||||
submitButton.disabled = isBusy;
|
||||
saveVoiceButton.disabled = isBusy;
|
||||
refreshVoicesButton.disabled = isBusy;
|
||||
deleteVoiceButton.disabled = isBusy;
|
||||
savePreviewButton.disabled = isBusy || !lastGeneratedBlob;
|
||||
refreshVectorsButton.disabled = isBusy;
|
||||
saveVectorButton.disabled = isBusy;
|
||||
deleteVectorButton.disabled = isBusy;
|
||||
}
|
||||
|
||||
function fillVoiceSelect(data) {
|
||||
const builtin = Array.isArray(data.voices) ? data.voices.map(String) : [];
|
||||
const uploadedEntries = Array.isArray(data.uploaded_voices) ? data.uploaded_voices : [];
|
||||
const uploaded = uploadedEntries
|
||||
.map((item) => typeof item === "string" ? { name: item } : item)
|
||||
.filter((item) => item && item.name)
|
||||
.map((item) => ({ ...item, name: String(item.name) }));
|
||||
const uploadedNames = new Set(uploaded.map((item) => item.name.toLowerCase()));
|
||||
const entries = [
|
||||
...uploaded.map((item) => ({
|
||||
name: item.name,
|
||||
label: `${item.name} (gespeichert, ${item.embedding_source || "audio"})`,
|
||||
uploaded: true,
|
||||
embedding_source: item.embedding_source || "audio"
|
||||
})),
|
||||
...builtin
|
||||
.filter((name) => !uploadedNames.has(name.toLowerCase()))
|
||||
.map((name) => ({ name, label: name, uploaded: false }))
|
||||
];
|
||||
|
||||
savedVoice.innerHTML = "";
|
||||
designVoice.innerHTML = "";
|
||||
for (const item of entries) {
|
||||
const option = document.createElement("option");
|
||||
option.value = item.name;
|
||||
option.textContent = item.label;
|
||||
option.dataset.uploaded = item.uploaded ? "true" : "false";
|
||||
option.dataset.embeddingSource = item.embedding_source || "";
|
||||
savedVoice.append(option);
|
||||
designVoice.append(option.cloneNode(true));
|
||||
}
|
||||
if (!entries.length) {
|
||||
const option = document.createElement("option");
|
||||
option.value = "";
|
||||
option.textContent = "Keine Stimmen gefunden";
|
||||
savedVoice.append(option);
|
||||
designVoice.append(option.cloneNode(true));
|
||||
}
|
||||
if ([...designVoice.options].some((option) => option.value === "ryan")) designVoice.value = "ryan";
|
||||
log(`Stimmen geladen: ${entries.length}`);
|
||||
}
|
||||
|
||||
async function loadVectors() {
|
||||
const endpoint = form.vectorsEndpoint.value.trim();
|
||||
const response = await fetch(endpoint);
|
||||
if (!response.ok) throw new Error(`GET ${endpoint} -> HTTP ${response.status}: ${await response.text()}`);
|
||||
const data = await response.json();
|
||||
const vectors = Array.isArray(data.vectors) ? data.vectors : [];
|
||||
vectorSelect.innerHTML = "";
|
||||
for (const item of vectors) {
|
||||
const option = document.createElement("option");
|
||||
option.value = item.name;
|
||||
option.textContent = `${item.name} (${item.dim})`;
|
||||
vectorSelect.append(option);
|
||||
}
|
||||
if (!vectors.length) {
|
||||
const option = document.createElement("option");
|
||||
option.value = "";
|
||||
option.textContent = "Keine Vectoren";
|
||||
vectorSelect.append(option);
|
||||
}
|
||||
log(`Vectoren geladen: ${vectors.length}`);
|
||||
}
|
||||
|
||||
async function loadConfig() {
|
||||
const response = await fetch("/api/config");
|
||||
if (!response.ok) return;
|
||||
const data = await response.json();
|
||||
if (data.model) form.model.value = data.model;
|
||||
}
|
||||
|
||||
async function loadVoices() {
|
||||
const endpoint = form.voicesEndpoint.value.trim();
|
||||
setStatus("Lade Stimmen");
|
||||
const response = await fetch(endpoint);
|
||||
if (!response.ok) {
|
||||
throw new Error(`GET ${endpoint} -> HTTP ${response.status}: ${await response.text()}`);
|
||||
}
|
||||
fillVoiceSelect(await response.json());
|
||||
setStatus("Stimmen geladen", "ok");
|
||||
}
|
||||
|
||||
async function saveVoice() {
|
||||
const file = refAudio.files[0];
|
||||
const name = form.voiceName.value.trim();
|
||||
const consent = form.consent.value.trim();
|
||||
if (!file || !name || !consent) {
|
||||
setStatus("Speichern unvollstaendig", "error");
|
||||
return;
|
||||
}
|
||||
|
||||
const data = new FormData();
|
||||
data.append("name", name);
|
||||
data.append("consent", consent);
|
||||
data.append("audio_sample", file, file.name);
|
||||
if (form.refText.value.trim()) data.append("ref_text", form.refText.value.trim());
|
||||
if (form.speakerDescription.value.trim()) data.append("speaker_description", form.speakerDescription.value.trim());
|
||||
|
||||
const endpoint = form.voicesEndpoint.value.trim();
|
||||
setBusy(true);
|
||||
setStatus("Speichere Stimme");
|
||||
try {
|
||||
log(`POST ${endpoint} name=${name}`);
|
||||
const response = await fetch(endpoint, { method: "POST", body: data });
|
||||
const text = await response.text();
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}: ${text.slice(0, 800)}`);
|
||||
log(text || "Stimme gespeichert");
|
||||
await loadVoices();
|
||||
savedVoice.value = name;
|
||||
sourceMode.value = "saved";
|
||||
syncMode();
|
||||
setStatus("Stimme gespeichert", "ok");
|
||||
} catch (error) {
|
||||
log(error && error.stack ? error.stack : String(error));
|
||||
setStatus("Fehler", "error");
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteVoice() {
|
||||
const name = savedVoice.value;
|
||||
if (!name) return;
|
||||
const endpoint = `${form.voicesEndpoint.value.trim()}/${encodeURIComponent(name)}`;
|
||||
setBusy(true);
|
||||
setStatus("Loesche Stimme");
|
||||
try {
|
||||
log(`DELETE ${endpoint}`);
|
||||
const response = await fetch(endpoint, { method: "DELETE" });
|
||||
const text = await response.text();
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}: ${text.slice(0, 800)}`);
|
||||
log(text || "Stimme geloescht");
|
||||
await loadVoices();
|
||||
setStatus("Stimme geloescht", "ok");
|
||||
} catch (error) {
|
||||
log(error && error.stack ? error.stack : String(error));
|
||||
setStatus("Fehler", "error");
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function buildSpeechPayload() {
|
||||
const payload = {
|
||||
model: form.model.value.trim(),
|
||||
input: form.inputText.value.trim(),
|
||||
language: form.language.value,
|
||||
response_format: "wav",
|
||||
instructions: form.instructions.value.trim(),
|
||||
speed: Number(form.speed.value || 1)
|
||||
};
|
||||
|
||||
if (sourceMode.value === "saved") {
|
||||
if (!savedVoice.value) throw new Error("Keine gespeicherte Stimme ausgewaehlt.");
|
||||
const selected = savedVoice.selectedOptions[0];
|
||||
const isUploadedAudio = selected?.dataset.uploaded === "true" && selected?.dataset.embeddingSource === "audio";
|
||||
const usesCustomVoiceModel = payload.model.includes("1.7B-CustomVoice");
|
||||
if (isUploadedAudio && form.voiceTaskType.value === "Base" && usesCustomVoiceModel) {
|
||||
throw new Error("Diese gespeicherte Audio-Stimme ist mit dem aktuell geladenen 1.7B-CustomVoice-Modell nicht testbar: der Server extrahiert 1024 Dimensionen, das Modell erwartet 2048. Nutze einen gespeicherten 2048er Speaker Vector oder starte ein kompatibles Base/Clone-Modell.");
|
||||
}
|
||||
payload.voice = savedVoice.value;
|
||||
payload.task_type = form.voiceTaskType.value;
|
||||
return payload;
|
||||
}
|
||||
|
||||
if (sourceMode.value === "vector") {
|
||||
if (!vectorSelect.value) throw new Error("Kein Speaker-Vector ausgewaehlt.");
|
||||
return payload;
|
||||
}
|
||||
|
||||
if (sourceMode.value === "design") {
|
||||
if (!designVoice.value) throw new Error("Keine Ausgangsstimme fuer Voice Design ausgewaehlt.");
|
||||
payload.voice = designVoice.value;
|
||||
payload.task_type = "VoiceDesign";
|
||||
payload.instructions = form.designDescription.value.trim();
|
||||
const seed = Number(form.designSeed.value);
|
||||
if (Number.isFinite(seed)) payload.seed = seed;
|
||||
return payload;
|
||||
}
|
||||
|
||||
const file = refAudio.files[0];
|
||||
if (!file) throw new Error("Fuer Ad-hoc-Cloning fehlt Referenzaudio.");
|
||||
const dataUrl = await fileToDataUrl(file);
|
||||
payload.task_type = "Base";
|
||||
payload.ref_audio = form.audioEncoding.value === "base64" ? dataUrl.split(",", 2)[1] : dataUrl;
|
||||
payload.ref_text = form.refText.value.trim();
|
||||
return payload;
|
||||
}
|
||||
|
||||
function syncMode() {
|
||||
savedVoiceBox.classList.toggle("hidden", sourceMode.value !== "saved");
|
||||
voiceDesignBox.classList.toggle("hidden", sourceMode.value !== "design");
|
||||
vectorBox.classList.toggle("hidden", sourceMode.value !== "vector");
|
||||
if (sourceMode.value === "saved") submitButton.textContent = "Stimme testen";
|
||||
else if (sourceMode.value === "design") submitButton.textContent = "Design erzeugen";
|
||||
else if (sourceMode.value === "vector") submitButton.textContent = "Mit Vector erzeugen";
|
||||
else submitButton.textContent = "Ad-hoc testen";
|
||||
}
|
||||
|
||||
refAudio.addEventListener("change", () => {
|
||||
const file = refAudio.files[0];
|
||||
fileMeta.textContent = file ? `${(file.size / 1024 / 1024).toFixed(2)} MB` : "Keine Datei";
|
||||
});
|
||||
|
||||
sourceMode.addEventListener("change", syncMode);
|
||||
refreshVoicesButton.addEventListener("click", () => loadVoices().catch((error) => {
|
||||
log(error && error.stack ? error.stack : String(error));
|
||||
setStatus("Fehler", "error");
|
||||
}));
|
||||
saveVoiceButton.addEventListener("click", saveVoice);
|
||||
deleteVoiceButton.addEventListener("click", deleteVoice);
|
||||
refreshVectorsButton.addEventListener("click", () => loadVectors().catch((error) => {
|
||||
log(error && error.stack ? error.stack : String(error));
|
||||
setStatus("Fehler", "error");
|
||||
}));
|
||||
saveVectorButton.addEventListener("click", async () => {
|
||||
setBusy(true);
|
||||
setStatus("Speichere Vector");
|
||||
try {
|
||||
const embedding = JSON.parse(form.vectorJson.value.trim());
|
||||
const response = await fetch(form.vectorsEndpoint.value.trim(), {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
name: form.vectorName.value.trim(),
|
||||
description: form.vectorDescription.value.trim(),
|
||||
embedding
|
||||
})
|
||||
});
|
||||
const text = await response.text();
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}: ${text.slice(0, 800)}`);
|
||||
log(text);
|
||||
await loadVectors();
|
||||
vectorSelect.value = form.vectorName.value.trim();
|
||||
setStatus("Vector gespeichert", "ok");
|
||||
} catch (error) {
|
||||
log(error && error.stack ? error.stack : String(error));
|
||||
setStatus("Fehler", "error");
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
});
|
||||
deleteVectorButton.addEventListener("click", async () => {
|
||||
if (!vectorSelect.value) return;
|
||||
setBusy(true);
|
||||
setStatus("Loesche Vector");
|
||||
try {
|
||||
const endpoint = `${form.vectorsEndpoint.value.trim()}/${encodeURIComponent(vectorSelect.value)}`;
|
||||
const response = await fetch(endpoint, { method: "DELETE" });
|
||||
const text = await response.text();
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}: ${text.slice(0, 800)}`);
|
||||
log(text);
|
||||
await loadVectors();
|
||||
setStatus("Vector geloescht", "ok");
|
||||
} catch (error) {
|
||||
log(error && error.stack ? error.stack : String(error));
|
||||
setStatus("Fehler", "error");
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
});
|
||||
savePreviewButton.addEventListener("click", async () => {
|
||||
if (!lastGeneratedBlob) return;
|
||||
const name = form.voiceName.value.trim();
|
||||
const consent = form.consent.value.trim();
|
||||
if (!name || !consent) {
|
||||
setStatus("Name oder Consent fehlt", "error");
|
||||
return;
|
||||
}
|
||||
const data = new FormData();
|
||||
data.append("name", name);
|
||||
data.append("consent", consent);
|
||||
data.append("audio_sample", lastGeneratedBlob, `${name}.wav`);
|
||||
data.append("ref_text", lastGeneratedText || form.inputText.value.trim());
|
||||
data.append("speaker_description", form.designDescription.value.trim() || form.speakerDescription.value.trim());
|
||||
const endpoint = form.voicesEndpoint.value.trim();
|
||||
setBusy(true);
|
||||
setStatus("Speichere Design");
|
||||
try {
|
||||
log(`POST ${endpoint} name=${name} aus Design-Vorschau`);
|
||||
const response = await fetch(endpoint, { method: "POST", body: data });
|
||||
const text = await response.text();
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}: ${text.slice(0, 800)}`);
|
||||
log(text || "Design als Stimme gespeichert");
|
||||
await loadVoices();
|
||||
savedVoice.value = name;
|
||||
sourceMode.value = "saved";
|
||||
syncMode();
|
||||
setStatus("Design gespeichert", "ok");
|
||||
} catch (error) {
|
||||
log(error && error.stack ? error.stack : String(error));
|
||||
setStatus("Fehler", "error");
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
});
|
||||
|
||||
resetButton.addEventListener("click", () => {
|
||||
form.reset();
|
||||
refAudio.value = "";
|
||||
fileMeta.textContent = "Keine Datei";
|
||||
logBox.textContent = "";
|
||||
resetResult();
|
||||
lastGeneratedBlob = null;
|
||||
lastGeneratedText = "";
|
||||
syncMode();
|
||||
setStatus("Bereit");
|
||||
loadVoices().catch((error) => log(String(error)));
|
||||
});
|
||||
|
||||
form.addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
resetResult();
|
||||
logBox.textContent = "";
|
||||
const endpoint = form.speechEndpoint.value.trim();
|
||||
setBusy(true);
|
||||
setStatus("Synthetisiere");
|
||||
|
||||
try {
|
||||
const payload = await buildSpeechPayload();
|
||||
log(`POST ${endpoint}`);
|
||||
log(`Quelle: ${sourceMode.value === "saved" ? payload.voice : "Ad-hoc Referenz"}`);
|
||||
log(`Task: ${payload.task_type}, Sprache: ${payload.language}`);
|
||||
const started = performance.now();
|
||||
const speechUrl = sourceMode.value === "vector"
|
||||
? `${form.vectorsEndpoint.value.trim()}/${encodeURIComponent(vectorSelect.value)}/speech`
|
||||
: endpoint;
|
||||
const response = await fetch(speechUrl, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(payload)
|
||||
});
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
throw new Error(`HTTP ${response.status}: ${text.slice(0, 800)}`);
|
||||
}
|
||||
const blob = await response.blob();
|
||||
lastGeneratedBlob = blob;
|
||||
lastGeneratedText = payload.input || "";
|
||||
currentObjectUrl = URL.createObjectURL(blob);
|
||||
player.src = currentObjectUrl;
|
||||
download.href = currentObjectUrl;
|
||||
download.hidden = false;
|
||||
savePreviewButton.disabled = false;
|
||||
const elapsed = ((performance.now() - started) / 1000).toFixed(2);
|
||||
log(`Fertig nach ${elapsed}s, ${(blob.size / 1024).toFixed(1)} KB`);
|
||||
setStatus("Fertig", "ok");
|
||||
} catch (error) {
|
||||
log(error && error.stack ? error.stack : String(error));
|
||||
setStatus("Fehler", "error");
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
});
|
||||
|
||||
syncMode();
|
||||
Promise.all([loadConfig(), loadVoices(), loadVectors()]).catch((error) => {
|
||||
log(error && error.stack ? error.stack : String(error));
|
||||
setStatus("Fehler", "error");
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
276
voice_clone_ui.py
Normal file
276
voice_clone_ui.py
Normal file
@@ -0,0 +1,276 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Serve the voice-cloning UI and proxy requests to the local Qwen3-TTS API."""
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from urllib import error, parse, request
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
UI_FILE = ROOT / "ui" / "voice-cloning.html"
|
||||
VECTOR_DIR = Path("/root/.cache/qwen3-tts-ui/vectors")
|
||||
NAME_RE = re.compile(r"^[A-Za-z0-9_.-]{1,80}$")
|
||||
|
||||
|
||||
class VoiceCloneHandler(BaseHTTPRequestHandler):
|
||||
api_base = "http://localhost:8091"
|
||||
clone_api_base = "http://localhost:8093"
|
||||
model = os.environ.get("QWEN3_TTS_MODEL", "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice")
|
||||
|
||||
def do_GET(self):
|
||||
if self.path in ("/", "/voice-cloning.html"):
|
||||
self._send_file(UI_FILE, "text/html; charset=utf-8")
|
||||
return
|
||||
if self.path == "/health":
|
||||
self._proxy_get("/health")
|
||||
return
|
||||
if self.path == "/api/config":
|
||||
self._json_response(200, {"model": self.model})
|
||||
return
|
||||
if self.path == "/api/v1/audio/voices":
|
||||
self._proxy_get("/v1/audio/voices")
|
||||
return
|
||||
if self.path == "/api/clone/health":
|
||||
self._proxy_get_base(self.clone_api_base, "/health", timeout=3)
|
||||
return
|
||||
if self.path == "/api/voice-vectors":
|
||||
self._list_vectors()
|
||||
return
|
||||
prefix = "/api/voice-vectors/"
|
||||
if self.path.startswith(prefix):
|
||||
self._get_vector(self.path[len(prefix):])
|
||||
return
|
||||
self.send_error(404)
|
||||
|
||||
def do_POST(self):
|
||||
if self.path == "/api/v1/audio/speech":
|
||||
self._proxy_post("/v1/audio/speech")
|
||||
return
|
||||
if self.path == "/api/v1/audio/voices":
|
||||
self._proxy_post("/v1/audio/voices")
|
||||
return
|
||||
if self.path == "/api/clone/speech":
|
||||
self._proxy_post_base(self.clone_api_base, "/v1/audio/speech", timeout=180)
|
||||
return
|
||||
if self.path == "/api/voice-vectors":
|
||||
self._save_vector()
|
||||
return
|
||||
prefix = "/api/voice-vectors/"
|
||||
if self.path.startswith(prefix) and self.path.endswith("/speech"):
|
||||
self._speech_with_vector(self.path[len(prefix):-len("/speech")])
|
||||
return
|
||||
self.send_error(404)
|
||||
|
||||
def do_DELETE(self):
|
||||
prefix = "/api/v1/audio/voices/"
|
||||
if self.path.startswith(prefix):
|
||||
name = parse.quote(parse.unquote(self.path[len(prefix):]), safe="")
|
||||
self._proxy_delete(f"/v1/audio/voices/{name}")
|
||||
return
|
||||
vector_prefix = "/api/voice-vectors/"
|
||||
if self.path.startswith(vector_prefix):
|
||||
self._delete_vector(self.path[len(vector_prefix):])
|
||||
return
|
||||
self.send_error(404)
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
print("%s - %s" % (self.address_string(), fmt % args))
|
||||
|
||||
|
||||
def _json_response(self, status, payload):
|
||||
body = json.dumps(payload).encode("utf-8")
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _read_json_body(self):
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
raw = self.rfile.read(length)
|
||||
return json.loads(raw.decode("utf-8") or "{}")
|
||||
|
||||
def _clean_vector_name(self, raw_name):
|
||||
name = parse.unquote(str(raw_name)).strip()
|
||||
if not NAME_RE.match(name):
|
||||
raise ValueError("Vector name must be 1-80 chars: letters, digits, dot, underscore or dash")
|
||||
return name
|
||||
|
||||
def _vector_path(self, raw_name):
|
||||
name = self._clean_vector_name(raw_name)
|
||||
return VECTOR_DIR / f"{name}.json"
|
||||
|
||||
def _validate_embedding(self, value):
|
||||
if isinstance(value, str):
|
||||
value = json.loads(value)
|
||||
if not isinstance(value, list) or not value:
|
||||
raise ValueError("embedding must be a non-empty list")
|
||||
if len(value) > 4096:
|
||||
raise ValueError("embedding exceeds 4096 values")
|
||||
emb = []
|
||||
for item in value:
|
||||
if not isinstance(item, (int, float)) or not math.isfinite(float(item)):
|
||||
raise ValueError("embedding must contain only finite numbers")
|
||||
emb.append(float(item))
|
||||
return emb
|
||||
|
||||
def _list_vectors(self):
|
||||
VECTOR_DIR.mkdir(parents=True, exist_ok=True)
|
||||
vectors = []
|
||||
for path in sorted(VECTOR_DIR.glob("*.json")):
|
||||
try:
|
||||
data = json.loads(path.read_text())
|
||||
vectors.append({
|
||||
"name": data.get("name", path.stem),
|
||||
"dim": len(data.get("embedding", [])),
|
||||
"description": data.get("description", ""),
|
||||
"created_at": data.get("created_at", 0),
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
self._json_response(200, {"vectors": vectors})
|
||||
|
||||
def _get_vector(self, raw_name):
|
||||
try:
|
||||
path = self._vector_path(raw_name)
|
||||
if not path.exists():
|
||||
self._json_response(404, {"error": "vector not found"})
|
||||
return
|
||||
self._json_response(200, json.loads(path.read_text()))
|
||||
except Exception as exc:
|
||||
self._json_response(400, {"error": str(exc)})
|
||||
|
||||
def _save_vector(self):
|
||||
try:
|
||||
data = self._read_json_body()
|
||||
name = self._clean_vector_name(data.get("name", ""))
|
||||
embedding = self._validate_embedding(data.get("embedding"))
|
||||
VECTOR_DIR.mkdir(parents=True, exist_ok=True)
|
||||
payload = {
|
||||
"name": name,
|
||||
"embedding": embedding,
|
||||
"description": str(data.get("description", "")),
|
||||
"created_at": data.get("created_at") or __import__("time").time(),
|
||||
}
|
||||
(VECTOR_DIR / f"{name}.json").write_text(json.dumps(payload))
|
||||
self._json_response(200, {"success": True, "vector": {"name": name, "dim": len(embedding)}})
|
||||
except Exception as exc:
|
||||
self._json_response(400, {"error": str(exc)})
|
||||
|
||||
def _delete_vector(self, raw_name):
|
||||
try:
|
||||
path = self._vector_path(raw_name)
|
||||
path.unlink(missing_ok=True)
|
||||
self._json_response(200, {"success": True})
|
||||
except Exception as exc:
|
||||
self._json_response(400, {"error": str(exc)})
|
||||
|
||||
def _speech_with_vector(self, raw_name):
|
||||
try:
|
||||
path = self._vector_path(raw_name)
|
||||
if not path.exists():
|
||||
self._json_response(404, {"error": "vector not found"})
|
||||
return
|
||||
vector = json.loads(path.read_text()).get("embedding")
|
||||
payload = self._read_json_body()
|
||||
payload["speaker_embedding"] = self._validate_embedding(vector)
|
||||
payload["task_type"] = "Base"
|
||||
payload["x_vector_only_mode"] = True
|
||||
body = json.dumps(payload).encode("utf-8")
|
||||
upstream = request.Request(
|
||||
f"{self.api_base}/v1/audio/speech",
|
||||
data=body,
|
||||
method="POST",
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
self._send_upstream(upstream)
|
||||
except Exception as exc:
|
||||
self._json_response(400, {"error": str(exc)})
|
||||
|
||||
def _send_file(self, path, content_type):
|
||||
if not path.exists():
|
||||
self.send_error(404)
|
||||
return
|
||||
body = path.read_bytes()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _proxy_get(self, target_path):
|
||||
self._proxy_get_base(self.api_base, target_path)
|
||||
|
||||
def _proxy_get_base(self, base_url, target_path, timeout=180):
|
||||
upstream = request.Request(f"{base_url}{target_path}", method="GET")
|
||||
self._send_upstream(upstream, timeout=timeout)
|
||||
|
||||
def _proxy_post(self, target_path):
|
||||
self._proxy_post_base(self.api_base, target_path)
|
||||
|
||||
def _proxy_post_base(self, base_url, target_path, timeout=180):
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
body = self.rfile.read(length)
|
||||
headers = {"Content-Type": self.headers.get("Content-Type", "application/json")}
|
||||
upstream = request.Request(
|
||||
f"{base_url}{target_path}",
|
||||
data=body,
|
||||
method="POST",
|
||||
headers=headers,
|
||||
)
|
||||
self._send_upstream(upstream, timeout=timeout)
|
||||
|
||||
def _proxy_delete(self, target_path):
|
||||
upstream = request.Request(f"{self.api_base}{target_path}", method="DELETE")
|
||||
self._send_upstream(upstream)
|
||||
|
||||
def _send_upstream(self, upstream, timeout=180):
|
||||
try:
|
||||
with request.urlopen(upstream, timeout=timeout) as response:
|
||||
body = response.read()
|
||||
self.send_response(response.status)
|
||||
self.send_header("Content-Type", response.headers.get("Content-Type", "application/octet-stream"))
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
except error.HTTPError as exc:
|
||||
body = exc.read()
|
||||
self.send_response(exc.code)
|
||||
self.send_header("Content-Type", exc.headers.get("Content-Type", "text/plain; charset=utf-8"))
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
except Exception as exc:
|
||||
body = json.dumps({"error": str(exc)}).encode("utf-8")
|
||||
self.send_response(502)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--host", default="127.0.0.1")
|
||||
parser.add_argument("--port", type=int, default=8092)
|
||||
parser.add_argument("--api-base", default="http://localhost:8091")
|
||||
parser.add_argument("--clone-api-base", default="http://localhost:8093")
|
||||
parser.add_argument("--model", default=os.environ.get("QWEN3_TTS_MODEL", "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"))
|
||||
args = parser.parse_args()
|
||||
|
||||
VoiceCloneHandler.api_base = args.api_base.rstrip("/")
|
||||
VoiceCloneHandler.clone_api_base = args.clone_api_base.rstrip("/")
|
||||
VoiceCloneHandler.model = args.model
|
||||
server = ThreadingHTTPServer((args.host, args.port), VoiceCloneHandler)
|
||||
print(f"UI: http://{args.host}:{args.port}/")
|
||||
print(f"Upstream: {VoiceCloneHandler.api_base}")
|
||||
print(f"Clone upstream: {VoiceCloneHandler.clone_api_base}")
|
||||
server.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
84
ws_log_proxy.py
Normal file
84
ws_log_proxy.py
Normal file
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Logging WebSocket proxy for Qwen3-TTS streaming calls."""
|
||||
import argparse
|
||||
import asyncio
|
||||
import datetime as dt
|
||||
import json
|
||||
import websockets
|
||||
|
||||
|
||||
def now():
|
||||
return dt.datetime.now().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def shorten(value, limit=500):
|
||||
text = value if isinstance(value, str) else repr(value)
|
||||
return text if len(text) <= limit else text[:limit] + "..."
|
||||
|
||||
|
||||
async def proxy(client_ws, upstream_url):
|
||||
conn_id = f"ws-{id(client_ws):x}"
|
||||
print(f"[{now()}] {conn_id} open -> {upstream_url}", flush=True)
|
||||
async with websockets.connect(upstream_url, max_size=None) as upstream_ws:
|
||||
async def client_to_upstream():
|
||||
async for msg in client_ws:
|
||||
if isinstance(msg, str):
|
||||
try:
|
||||
data = json.loads(msg)
|
||||
typ = data.get("type")
|
||||
if typ == "session.config":
|
||||
log_data = dict(data)
|
||||
if "speaker_embedding" in log_data:
|
||||
emb = log_data["speaker_embedding"] or []
|
||||
log_data["speaker_embedding"] = f"<{len(emb)} floats>"
|
||||
print(f"[{now()}] {conn_id} C->S config {json.dumps(log_data, ensure_ascii=False, sort_keys=True)}", flush=True)
|
||||
elif typ == "input.text":
|
||||
print(f"[{now()}] {conn_id} C->S text {data.get('text')!r}", flush=True)
|
||||
else:
|
||||
print(f"[{now()}] {conn_id} C->S {shorten(msg)}", flush=True)
|
||||
except Exception:
|
||||
print(f"[{now()}] {conn_id} C->S {shorten(msg)}", flush=True)
|
||||
else:
|
||||
print(f"[{now()}] {conn_id} C->S binary {len(msg)} bytes", flush=True)
|
||||
await upstream_ws.send(msg)
|
||||
|
||||
async def upstream_to_client():
|
||||
async for msg in upstream_ws:
|
||||
if isinstance(msg, str):
|
||||
try:
|
||||
data = json.loads(msg)
|
||||
print(f"[{now()}] {conn_id} S->C {data}", flush=True)
|
||||
except Exception:
|
||||
print(f"[{now()}] {conn_id} S->C {shorten(msg)}", flush=True)
|
||||
else:
|
||||
print(f"[{now()}] {conn_id} S->C binary {len(msg)} bytes", flush=True)
|
||||
await client_ws.send(msg)
|
||||
|
||||
await asyncio.gather(client_to_upstream(), upstream_to_client())
|
||||
|
||||
|
||||
async def handler(client_ws):
|
||||
try:
|
||||
await proxy(client_ws, handler.upstream_url)
|
||||
except Exception as exc:
|
||||
print(f"[{now()}] proxy error: {exc}", flush=True)
|
||||
try:
|
||||
await client_ws.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--host", default="0.0.0.0")
|
||||
parser.add_argument("--port", type=int, default=8094)
|
||||
parser.add_argument("--upstream", default="ws://localhost:8091/v1/audio/speech/stream")
|
||||
args = parser.parse_args()
|
||||
handler.upstream_url = args.upstream
|
||||
print(f"WS log proxy: ws://{args.host}:{args.port}/v1/audio/speech/stream -> {args.upstream}", flush=True)
|
||||
async with websockets.serve(handler, args.host, args.port, max_size=None):
|
||||
await asyncio.Future()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user