From 94cbf29972ede5bb75594dc53c1056723e31bbcf Mon Sep 17 00:00:00 2001 From: "Wolf G. Beckmann" Date: Wed, 17 Jun 2026 08:33:47 +0200 Subject: [PATCH] first commit --- STREAMING_API.md | 147 +++++++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 24 ++++++++ entrypoint.sh | 41 +++++++++++++ start.sh | 69 +++++++++++++++++++++ test_http.py | 62 +++++++++++++++++++ test_ws.py | 93 ++++++++++++++++++++++++++++ 6 files changed, 436 insertions(+) create mode 100644 STREAMING_API.md create mode 100644 docker-compose.yml create mode 100755 entrypoint.sh create mode 100755 start.sh create mode 100644 test_http.py create mode 100644 test_ws.py diff --git a/STREAMING_API.md b/STREAMING_API.md new file mode 100644 index 0000000..33edcce --- /dev/null +++ b/STREAMING_API.md @@ -0,0 +1,147 @@ +# Qwen3-TTS — Streaming-API (Token-Stream → Audio) + +WebSocket-Endpoint, um Text **inkrementell** (z. B. Token für Token aus einem LLM) +zu senden und Audio **satzweise und chunked** zurückzubekommen. Der Server puffert +den einströmenden Text, segmentiert ihn an Satz-/Teilsatzgrenzen und synthetisiert +jeden Abschnitt, sobald er vollständig ist. + +## Endpunkte & Auth + +| | Lokal | Extern | +|---|---|---| +| WebSocket | `ws://localhost:8091/v1/audio/speech/stream` | `wss://qwen3-tts.aquantico.de/v1/audio/speech/stream` | +| REST (Vollausgabe) | `http://localhost:8091/v1/audio/speech` | `https://qwen3-tts.aquantico.de/v1/audio/speech` | +| Health / Modelle | `…/health`, `…/v1/models` | dito | + +- **Auth (nur extern):** Header `Authorization: Bearer `. Lokal ist kein Token nötig. +- **TLS (extern):** Das Zertifikat ist self-signed → Client-Zertifikatsprüfung ggf. deaktivieren + (`curl -k`, bzw. `ssl.CERT_NONE` in Python). Intern kein TLS. +- **Audioformat:** 24 000 Hz, mono, 16-bit PCM (little-endian). Bei `response_format: "wav"` + kommt pro Satz eine komplette WAV-Datei; bei `"pcm"` rohe PCM-Chunks (ohne Header). + +## Protokoll + +Eine WebSocket-Verbindung = eine Session. Ablauf: + +### Client → Server (JSON-Textframes) + +1. **`session.config`** — genau einmal, als allererste Nachricht (Timeout 10 s): + ```json + { + "type": "session.config", + "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", + "voice": "Ryan", + "language": "German", + "response_format": "pcm", + "stream_audio": true, + "split_granularity": "sentence" + } + ``` +2. **`input.text`** — beliebig viele; je Nachricht ein Token/Fragment/Satz (max. 128 KB je Frame): + ```json + {"type": "input.text", "text": "Hallo"} + {"type": "input.text", "text": ", wie geht es "} + {"type": "input.text", "text": "Ihnen?"} + ``` + Der Server hängt alles an einen Puffer an und löst Synthese aus, **sobald eine + Satz-/Teilsatzgrenze erkannt wird**. Unvollständige Reste bleiben gepuffert. + Max. 30 s Pause zwischen zwei Nachrichten (idle timeout). +3. **`input.done`** — Eingabe beendet; verbleibender Pufferinhalt wird noch synthetisiert, + dann folgt `session.done`. + +### Server → Client + +| Nachricht | Inhalt | +|---|---| +| `audio.start` (JSON) | `{"type":"audio.start","sentence_index":0,"sentence_text":"…","format":"pcm","sample_rate":24000}` — Beginn eines Satzes. `sample_rate` nur bei PCM-Streaming. | +| *(Binärframe)* | Audio-Bytes. Bei `stream_audio:true` + `pcm` mehrere Chunks, die schon **während** der Generierung kommen. Bei `wav` ein vollständiger WAV-Block. | +| `audio.done` (JSON) | `{"type":"audio.done","sentence_index":0}` — Satz fertig. | +| `session.done` (JSON) | `{"type":"session.done","total_sentences":N}` — Session fertig. | +| `error` (JSON) | `{"type":"error","message":"…"}` | + +## `session.config` — Felder + +| Feld | Default | Beschreibung | +|---|---|---| +| `model` | — | Modell-ID (wird validiert). | +| `voice` | (modellabh.) | Eingebauter Sprecher, z. B. `Ryan`, `Aiden`, `Vivian` (CustomVoice). | +| `language` | `Auto` | z. B. `German`, `English`. | +| `task_type` | `CustomVoice` | `CustomVoice` \| `VoiceDesign` \| `Base`. | +| `instructions` | — | Natürlichsprachliche Stil-/Emotionssteuerung (z. B. „sachlich, ruhig"). | +| `response_format` | `wav` | `pcm` (für echtes Low-Latency-Streaming empfohlen) oder `wav`. | +| `stream_audio` | `false` | `true` = Audio-Chunks während der Generierung; `false` = ganzer Satz auf einmal. | +| `split_granularity` | `sentence` | `sentence` oder `clause` (an Kommas etc. → noch kleinere Stücke, niedrigere Latenz). | +| `speed` | `1.0` | Sprechtempo. | +| `max_new_tokens` | (modellabh.) | Obergrenze Token. | +| `initial_codec_chunk_frames` | (deploy-cfg) | Größe des ersten Audio-Chunks (Latenz-Tuning). | +| `ref_audio` / `ref_text` | — | Referenz für Voice-Clone (`Base`-Task; nur mit klon-fähigem Modell). | +| `speaker_embedding` | — | Fester 2048-dim-Stimmvektor (1.7B) für konstantes Timbre. | +| `x_vector_only_mode` | — | Klon nur über x-Vector (ohne ICL-Transkript). | + +## Segmentierung & Stimm-Konsistenz + +- Synthese erfolgt **pro Segment** (Satz bzw. Teilsatz). `split_granularity: "clause"` + senkt die Latenz, erzeugt aber mehr Segmentgrenzen. +- ⚠️ **Drift:** Jedes Segment wird unabhängig generiert; Stimmlage/Stimmung können zwischen + Segmenten leicht schwanken (gemessen ~11 % F0-Streuung). Temperatur senken hilft **nicht**. + Gegenmittel: (a) mehr Text pro Segment bündeln, (b) Timbre über einen festen + `speaker_embedding` verankern (Base-Modell nötig). + +## Gemessene Latenz + +| Pfad | Time-to-first-audio | Hinweis | +|---|---|---| +| Lokal, 1 Stream | ~0,16 s | ab Satzende | +| Lokal, 8 parallele Streams | ~0,23 s | TTFT lastunabhängig stabil | +| Extern (WSS, Token-Stream à 0,12 s) | ~1,6 s gesamt | inkl. Token-Eintrudeln + Netz; ~0,3 s ab Satzende | + +## Python-Beispiel (Token-Stream → PCM) + +```python +import asyncio, json, ssl, inspect, websockets + +WSS = "wss://qwen3-tts.aquantico.de/v1/audio/speech/stream" +TOKEN = "" + +def connect(): + ctx = ssl.create_default_context() + ctx.check_hostname = False # self-signed cert + ctx.verify_mode = ssl.CERT_NONE + hdr = {"Authorization": f"Bearer {TOKEN}"} + params = inspect.signature(websockets.connect).parameters + key = "additional_headers" if "additional_headers" in params else "extra_headers" + return websockets.connect(WSS, max_size=None, ssl=ctx, **{key: hdr}) + +async def speak(token_iter): + async with connect() as ws: + await ws.send(json.dumps({ + "type": "session.config", + "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", + "voice": "Ryan", "language": "German", + "response_format": "pcm", "stream_audio": True, + })) + + async def reader(): + async for m in ws: + if isinstance(m, (bytes, bytearray)): + handle_pcm(m) # an Player/Buffer geben (24kHz mono s16le) + else: + msg = json.loads(m) + if msg["type"] == "session.done": return + if msg["type"] == "error": raise RuntimeError(msg["message"]) + + rt = asyncio.create_task(reader()) + for tok in token_iter: # Tokens direkt aus dem LLM + await ws.send(json.dumps({"type": "input.text", "text": tok})) + await ws.send(json.dumps({"type": "input.done"})) + await rt + +def handle_pcm(chunk: bytes): + ... # z.B. sounddevice / Datei / Web-Client +``` + +## Wichtige Limits + +- `session.config` muss innerhalb **10 s** kommen, sonst Abbruch. +- Max. **30 s** Pause zwischen Nachrichten (idle timeout). +- `session.config` ≤ 4 MB (für große `ref_audio`-Payloads), `input.text` ≤ 128 KB pro Frame. diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..94d5084 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,24 @@ +services: + qwen3-tts: + image: vllm/vllm-omni:latest-aarch64 + container_name: qwen3-tts + runtime: nvidia + environment: + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=compute,utility + - VLLM_LOGGING_LEVEL=INFO + - HF_HOME=/root/.cache/huggingface + ports: + - "8091:8091" + volumes: + - /home/guru/vllm/data/root:/root + - ./deploy/qwen3_tts.yaml:/deploy/qwen3_tts.yaml:ro + - ./entrypoint.sh:/entrypoint.sh:ro + command: ["/bin/bash", "/entrypoint.sh"] + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:8091/health"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 120s diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..61e3ade --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Container-Entrypoint: startet den vLLM-Omni TTS-Server und feuert nach +# Erreichen von /health einmalig einen Warmup-Request ab. Dieser absorbiert +# die einmalige torch.compile/CUDA-Graph-Kompilierung (~38 s), damit kein +# echter Nutzer-Request sie je trifft. Läuft bei JEDEM Containerstart +# (Reboot, Crash-Restart, manuell). +set -e + +MODEL="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice" +PORT=8091 + +# --- Server im Hintergrund starten --- +vllm-omni serve "$MODEL" \ + --omni \ + --host 0.0.0.0 \ + --port "$PORT" \ + --deploy-config /deploy/qwen3_tts.yaml \ + --gpu-memory-utilization 0.10 \ + --trust-remote-code & +SERVER_PID=$! + +# --- Warmup im Hintergrund, sobald der Server gesund ist --- +( + for _ in $(seq 1 150); do + if curl -sf "http://localhost:${PORT}/health" >/dev/null 2>&1; then + echo "[warmup] Server gesund — sende Warmup-Request (kompiliert CUDA-Graphen)..." + t0=$SECONDS + curl -sf -X POST "http://localhost:${PORT}/v1/audio/speech" \ + -H 'Content-Type: application/json' \ + -d "{\"model\":\"${MODEL}\",\"input\":\"System wird aufgewärmt.\",\"voice\":\"Ryan\",\"language\":\"German\",\"response_format\":\"wav\"}" \ + -o /dev/null \ + && echo "[warmup] fertig nach $(( SECONDS - t0 ))s — Server ist jetzt heiß." \ + || echo "[warmup] fehlgeschlagen (Server läuft trotzdem)." + break + fi + sleep 2 + done +) & + +# --- Auf den Server-Prozess warten (PID 1 Signalweiterleitung) --- +wait "$SERVER_PID" diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..2871902 --- /dev/null +++ b/start.sh @@ -0,0 +1,69 @@ +#!/bin/bash +set -e +cd "$(dirname "$0")" + +HEALTH_URL="http://localhost:8091/health" +MAX_WAIT=180 + +strip_ansi() { sed 's/\x1b\[[0-9;]*[mK]//g'; } + +t0=$SECONDS + +# --- Container starten --- +printf "Container startet..." +docker compose up -d &>/tmp/qwen3-tts-compose.log +printf " [%ds]\n\n" $(( SECONDS - t0 )) + +# --- Log-Events im Hintergrund parsen und ausgeben --- +docker logs -f qwen3-tts 2>&1 | strip_ansi | while IFS= read -r line; do + case "$line" in + *"Stage 0 engine launch started"*) + printf " Stage 0 Talker LLM wird geladen\n" ;; + *"Initializing a V1 LLM engine"*) + printf " Stage 0 Engine initialisiert\n" ;; + *"Loading weights"*) + printf " Stage 0 Modell-Gewichte werden geladen...\n" ;; + *"Loading model weights took"*) + took=$(echo "$line" | grep -oP '\d+\.\d+s' | head -1) + printf " Stage 0 Gewichte geladen (%s)\n" "$took" ;; + *"Stage 1 engine launch started"*) + printf " Stage 1 Code2Wav Decoder wird geladen\n" ;; + *"Graph capturing finished"*) + printf " CUDA Graphen kompiliert\n" ;; + *"Application startup complete"*) + printf " Server FastAPI bereit\n" ;; + *"Uvicorn running on"*) + printf " Server HTTP läuft auf Port 8091\n" ;; + *"warmup] Server gesund"*) + printf " Warmup Aufwärm-Request läuft (kompiliert/lädt Graphen)...\n" ;; + *"warmup] fertig"*) + took=$(echo "$line" | grep -oP '\d+s' | head -1) + printf " Warmup Server ist heiß (%s)\n" "$took" ;; + *"ERROR"*|*"ValueError"*) + msg=$(echo "$line" | grep -oP '(ERROR|ValueError).*' | head -1) + printf " [!] %s\n" "${msg:0:100}" ;; + esac +done & +LOG_PID=$! + +# --- Health-Check-Schleife --- +while (( SECONDS - t0 < MAX_WAIT )); do + if curl -sf "$HEALTH_URL" >/dev/null 2>&1; then + kill "$LOG_PID" 2>/dev/null + wait "$LOG_PID" 2>/dev/null + printf "\n✓ Bereit: http://localhost:8091 [%ds]\n" $(( SECONDS - t0 )) + exit 0 + fi + # Abbruch wenn Container schon gestoppt ist + if ! docker ps --filter "name=qwen3-tts" --filter "status=running" -q | grep -q .; then + kill "$LOG_PID" 2>/dev/null + printf "\n✗ Container abgestürzt. Logs:\n" + docker logs --tail=20 qwen3-tts 2>&1 | strip_ansi | grep -E "ERROR|ValueError|Exception" | head -5 + exit 1 + fi + sleep 2 +done + +kill "$LOG_PID" 2>/dev/null +printf "\n✗ Timeout nach %ds. Logs: docker logs qwen3-tts\n" "$MAX_WAIT" >&2 +exit 1 diff --git a/test_http.py b/test_http.py new file mode 100644 index 0000000..98cbd87 --- /dev/null +++ b/test_http.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""REST test for Qwen3-TTS /v1/audio/speech endpoint.""" +import sys +import time +import requests + +BASE_URL = "http://localhost:8091" +OUTPUT_FILE = "test_output.wav" + + +def check_health(): + try: + r = requests.get(f"{BASE_URL}/health", timeout=5) + r.raise_for_status() + body = r.json() if r.content else "(leer)" + print(f"[OK] /health → HTTP {r.status_code} {body}") + return True + except Exception as e: + print(f"[FAIL] /health: {e}") + return False + + +def test_tts(text: str, voice: str = "Ryan", language: str = "German"): + payload = { + "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", + "input": text, + "voice": voice, + "language": language, + "response_format": "wav", + } + print(f"\n[→] POST /v1/audio/speech") + print(f" text: {text!r}") + print(f" voice: {voice}, language: {language}") + + t0 = time.perf_counter() + r = requests.post(f"{BASE_URL}/v1/audio/speech", json=payload, timeout=60) + elapsed = time.perf_counter() - t0 + + if r.status_code != 200: + print(f"[FAIL] HTTP {r.status_code}: {r.text[:300]}") + sys.exit(1) + + with open(OUTPUT_FILE, "wb") as f: + f.write(r.content) + + kb = len(r.content) / 1024 + print(f"[OK] {elapsed:.2f}s → {kb:.1f} KB WAV saved to {OUTPUT_FILE}") + + +if __name__ == "__main__": + print(f"Testing Qwen3-TTS at {BASE_URL}\n") + + if not check_health(): + sys.exit(1) + + test_tts( + text="Hallo! Ich bin ein KI-Sprachassistent. Wie kann ich Ihnen helfen?", + voice="Ryan", + language="German", + ) + + print("\nAll tests passed.") diff --git a/test_ws.py b/test_ws.py new file mode 100644 index 0000000..0a4759a --- /dev/null +++ b/test_ws.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""WebSocket streaming test for Qwen3-TTS /v1/audio/speech/stream endpoint.""" +import sys +import json +import time +import wave +import struct +import asyncio + +try: + import websockets +except ImportError: + print("Installing websockets...") + import subprocess + subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets"]) + import websockets + +WS_URL = "ws://localhost:8091/v1/audio/speech/stream" +OUTPUT_FILE = "test_stream_output.wav" +SAMPLE_RATE = 24000 + + +async def stream_tts(text: str, voice: str = "Ryan", language: str = "German"): + print(f"[→] WebSocket connect: {WS_URL}") + print(f" text: {text!r}") + print(f" voice: {voice}, language: {language}") + + pcm_chunks = [] + total_bytes = 0 + t0 = time.perf_counter() + first_chunk_time = None + + async with websockets.connect(WS_URL) as ws: + # Send session config + await ws.send(json.dumps({ + "type": "session.config", + "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", + "task_type": "CustomVoice", + "voice": voice, + "language": language, + "response_format": "pcm", + "stream_audio": True, + })) + + # Send text + await ws.send(json.dumps({ + "type": "input.text", + "text": text, + })) + await ws.send(json.dumps({"type": "input.done"})) + + # Receive + async for message in ws: + if isinstance(message, bytes): + if first_chunk_time is None: + first_chunk_time = time.perf_counter() - t0 + print(f"[OK] First audio chunk in {first_chunk_time:.3f}s") + pcm_chunks.append(message) + total_bytes += len(message) + elif isinstance(message, str): + msg = json.loads(message) + msg_type = msg.get("type", "") + if msg_type == "session.done": + break + elif msg_type == "error": + print(f"[FAIL] Server error: {msg.get('message')}") + sys.exit(1) + elif msg_type in ("audio.start", "audio.done"): + print(f" [{msg_type}] sentence {msg.get('sentence_index', '?')}") + + elapsed = time.perf_counter() - t0 + + # Write PCM chunks as WAV + raw_pcm = b"".join(pcm_chunks) + with wave.open(OUTPUT_FILE, "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) # int16 + wf.setframerate(SAMPLE_RATE) + wf.writeframes(raw_pcm) + + audio_duration = (total_bytes / 2) / SAMPLE_RATE + print(f"[OK] Done: {elapsed:.2f}s total, {total_bytes/1024:.1f} KB PCM") + print(f" Audio duration: {audio_duration:.2f}s → saved to {OUTPUT_FILE}") + if first_chunk_time: + print(f" First-chunk latency: {first_chunk_time:.3f}s") + + +if __name__ == "__main__": + asyncio.run(stream_tts( + text="Herzlich willkommen! Dieses System nutzt Qwen3-TTS für deutsche Sprachausgabe.", + voice="Ryan", + language="German", + ))