Qwen3-tts/docker/ws_log_proxy.py

#!/usr/bin/env python3
"""Combined HTTP + WebSocket proxy for Qwen3-TTS on a single port.

Routes on one port (default 8091):
  POST /v1/audio/speech          → REST proxy to internal vllm-omni
  GET  /v1/audio/speech/stream   → WebSocket (PCM-paced, REST-sourced)
  GET  /health                   → health proxy to internal vllm-omni

Why REST-sourced WebSocket?
  The vllm-omni streaming codec drops the onset of short utterances.
  The batch REST endpoint (/v1/audio/speech) uses full decode and renders
  every word cleanly. This proxy keeps the client-facing WS protocol intact
  while sourcing audio sentence-by-sentence from REST, pacing the result
  into one continuous real-time PCM stream via pcm_pacer.

Why instructions is NOT forwarded:
  With a cloned ICL voice (ref_audio) the voice character comes from the
  reference; an extra style prompt makes the Talker emit EOS far too early
  and clips short words ("Hallo." → 0.16 s vs 0.40 s without).
"""
import argparse
import asyncio
import datetime as dt
import json
import math
import re

import aiohttp
import websockets
from aiohttp import web

# ---------------------------------------------------------------------------
SAMPLE_RATE      = 24_000
BYTES_PER_SAMPLE = 2
BYTES_PER_SEC    = SAMPLE_RATE * BYTES_PER_SAMPLE   # 48 000

SEND_CHUNK   = 960                                   # 20 ms at 48 kHz
FRAME_SECS   = SEND_CHUNK / BYTES_PER_SEC            # 0.020 s

LEADING_MS      = 300
LEADING_FRAMES  = math.ceil(LEADING_MS  / 1000 / FRAME_SECS)
TRAILING_MS     = 2_000
TRAILING_FRAMES = math.ceil(TRAILING_MS / 1000 / FRAME_SECS)

# Per-sentence synthesis timeout. A healthy sentence renders in <2 s.
SYNTH_TIMEOUT_S      = 30
# Direct REST proxy timeout — longer for potentially bigger inputs.
REST_PROXY_TIMEOUT_S = 120

SILENCE_CHUNK = bytes(SEND_CHUNK)

SPLIT_RE         = re.compile(r"(?<=[.!?])\s+|(?<=[。！？])")
MIN_SENTENCE_LEN = 2

# Fields forwarded to the internal REST synthesis request.
# `instructions` deliberately omitted — see module docstring.
REST_PASSTHROUGH = ("model", "voice", "task_type", "language",
                    "seed", "speaker_embedding")
# ---------------------------------------------------------------------------

_internal_base_url: str = ""   # e.g. http://localhost:8095
_internal_ws_url:   str = ""   # e.g. ws://localhost:8095/v1/audio/speech/stream


def now() -> str:
    return dt.datetime.now().isoformat(timespec="seconds")


class SentenceSplitter:
    """Incremental splitter: buffers text, yields complete sentences."""

    def __init__(self) -> None:
        self.buf = ""

    def add(self, text: str) -> list[str]:
        self.buf += text
        out: list[str] = []
        while True:
            m = SPLIT_RE.search(self.buf)
            if not m:
                break
            idx  = m.end()
            head = self.buf[:idx].strip()
            if len(head) < MIN_SENTENCE_LEN:
                break
            self.buf = self.buf[idx:]
            if head:
                out.append(head)
        return out

    def flush(self) -> list[str]:
        s = self.buf.strip()
        self.buf = ""
        return [s] if s else []


async def synth_rest(http: aiohttp.ClientSession, cfg: dict, text: str) -> bytes:
    """Synthesise one sentence via the internal batch REST endpoint → raw PCM."""
    payload = {"input": text, "response_format": "pcm"}
    for k in REST_PASSTHROUGH:
        if cfg.get(k) is not None:
            payload[k] = cfg[k]
    async with http.post(f"{_internal_base_url}/v1/audio/speech",
                         json=payload) as resp:
        resp.raise_for_status()
        return await resp.read()


# ── REST proxy ───────────────────────────────────────────────────────────────

async def handle_rest(request: web.Request) -> web.StreamResponse:
    """Forward POST /v1/audio/speech to the internal vllm-omni REST API."""
    body    = await request.read()
    ct      = request.headers.get("Content-Type", "application/json")
    timeout = aiohttp.ClientTimeout(total=REST_PROXY_TIMEOUT_S)
    async with aiohttp.ClientSession(timeout=timeout) as http:
        async with http.post(
            f"{_internal_base_url}/v1/audio/speech",
            data=body,
            headers={"Content-Type": ct},
        ) as resp:
            out_ct   = resp.headers.get("Content-Type", "application/octet-stream")
            response = web.StreamResponse(
                status=resp.status,
                headers={"Content-Type": out_ct},
            )
            await response.prepare(request)
            async for chunk in resp.content.iter_chunked(65536):
                await response.write(chunk)
            await response.write_eof()
            return response


async def handle_health(request: web.Request) -> web.Response:
    """Proxy GET /health to internal vllm-omni."""
    try:
        async with aiohttp.ClientSession() as http:
            async with http.get(f"{_internal_base_url}/health") as resp:
                return web.Response(status=resp.status, text=await resp.text())
    except Exception:
        return web.Response(status=503, text="unavailable")


# ── WebSocket PCM session ────────────────────────────────────────────────────

async def proxy_session_pcm(client_ws: web.WebSocketResponse,
                             conn_id: str, cfg: dict) -> None:
    """REST-sourced, paced, single continuous PCM stream per session."""
    pcm_queue:      asyncio.Queue = asyncio.Queue()
    sentence_queue: asyncio.Queue = asyncio.Queue()
    splitter = SentenceSplitter()
    started  = {"audio": False}

    def enqueue_pcm(data: bytes) -> None:
        for i in range(0, len(data), SEND_CHUNK):
            piece = data[i:i + SEND_CHUNK]
            if len(piece) < SEND_CHUNK:
                piece = piece + bytes(SEND_CHUNK - len(piece))
            pcm_queue.put_nowait(piece)

    async def pcm_pacer() -> bool:
        """Drain pcm_queue at 48 kHz. Returns True = fully drained, False = disconnected."""
        loop = asyncio.get_event_loop()
        t0   = loop.time()
        sent = 0
        try:
            while True:
                target = t0 + sent / BYTES_PER_SEC
                delay  = target - loop.time()
                if delay > 0.001:
                    await asyncio.sleep(delay)
                try:
                    chunk = pcm_queue.get_nowait()
                    if chunk is None:
                        break
                except asyncio.QueueEmpty:
                    chunk = SILENCE_CHUNK
                await client_ws.send_bytes(chunk)
                sent += len(chunk)
        except Exception as exc:
            print(f"[{now()}] {conn_id} PACER send failed after {sent} B "
                  f"({sent/BYTES_PER_SEC:.2f}s): {exc!r}", flush=True)
            return False
        print(f"[{now()}] {conn_id} PACER done, {sent} B "
              f"({sent/BYTES_PER_SEC:.2f}s)", flush=True)
        return True

    pacer_task = asyncio.create_task(pcm_pacer())

    async def synth_worker() -> None:
        timeout = aiohttp.ClientTimeout(total=SYNTH_TIMEOUT_S)
        async with aiohttp.ClientSession(timeout=timeout) as http:
            while True:
                text = await sentence_queue.get()
                if text is None:
                    break
                try:
                    pcm = await synth_rest(http, cfg, text)
                except Exception as exc:
                    print(f"[{now()}] {conn_id} REST synth failed for "
                          f"{text!r}: {exc!r}", flush=True)
                    continue
                print(f"[{now()}] {conn_id} REST {text!r} -> {len(pcm)} B "
                      f"({len(pcm)/BYTES_PER_SEC:.2f}s)", flush=True)
                if not started["audio"]:
                    started["audio"] = True
                    await client_ws.send_str(json.dumps(
                        {"type": "audio.start", "format": "pcm",
                         "sample_rate": SAMPLE_RATE}))
                    for _ in range(LEADING_FRAMES):
                        pcm_queue.put_nowait(SILENCE_CHUNK)
                enqueue_pcm(pcm)
        for _ in range(TRAILING_FRAMES):
            pcm_queue.put_nowait(SILENCE_CHUNK)
        pcm_queue.put_nowait(None)

    worker_task = asyncio.create_task(synth_worker())

    async def cleanup() -> None:
        for t in (worker_task, pacer_task):
            if not t.done():
                t.cancel()
        await asyncio.gather(worker_task, pacer_task, return_exceptions=True)

    try:
        async for msg in client_ws:
            if msg.type == aiohttp.WSMsgType.TEXT:
                try:
                    data = json.loads(msg.data)
                    typ  = data.get("type")
                except Exception:
                    continue
                if typ == "input.text":
                    text = data.get("text", "")
                    print(f"[{now()}] {conn_id} C text {text!r}", flush=True)
                    for sent in splitter.add(text):
                        sentence_queue.put_nowait(sent)
                elif typ == "input.done":
                    print(f"[{now()}] {conn_id} C input.done", flush=True)
                    for sent in splitter.flush():
                        sentence_queue.put_nowait(sent)
                    sentence_queue.put_nowait(None)
                    drained_ok = await pacer_task
                    if not worker_task.done():
                        worker_task.cancel()
                    if drained_ok:
                        try:
                            await client_ws.send_str(
                                json.dumps({"type": "session.done"}))
                        except Exception:
                            pass
                    return
            elif msg.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.ERROR):
                break
    finally:
        await cleanup()


async def proxy_session_passthrough(client_ws: web.WebSocketResponse,
                                    conn_id: str, first_msg_data: str) -> None:
    """Transparent WS passthrough to streaming upstream (non-PCM clients)."""
    async with websockets.connect(_internal_ws_url, max_size=None) as upstream:
        await upstream.send(first_msg_data)

        async def c2u() -> None:
            async for msg in client_ws:
                if msg.type == aiohttp.WSMsgType.TEXT:
                    await upstream.send(msg.data)
                elif msg.type == aiohttp.WSMsgType.BINARY:
                    await upstream.send(msg.data)
                else:
                    break

        async def u2c() -> None:
            async for m in upstream:
                if isinstance(m, str):
                    await client_ws.send_str(m)
                else:
                    await client_ws.send_bytes(m)

        await asyncio.gather(c2u(), u2c())


async def handle_ws(request: web.Request) -> web.WebSocketResponse:
    conn_id   = f"ws-{id(request):x}"
    client_ws = web.WebSocketResponse(max_msg_size=0)
    await client_ws.prepare(request)
    print(f"[{now()}] {conn_id} open", flush=True)

    # First client message must be session.config.
    first = await client_ws.receive()
    if first.type != aiohttp.WSMsgType.TEXT:
        return client_ws
    try:
        cfg = json.loads(first.data)
    except Exception:
        cfg = {}

    if cfg.get("response_format", "pcm") == "pcm":
        log = dict(cfg)
        if log.get("speaker_embedding"):
            log["speaker_embedding"] = f"<{len(log['speaker_embedding'])} floats>"
        print(f"[{now()}] {conn_id} config "
              f"{json.dumps(log, ensure_ascii=False, sort_keys=True)}", flush=True)
        await proxy_session_pcm(client_ws, conn_id, cfg)
    else:
        print(f"[{now()}] {conn_id} non-PCM -> passthrough", flush=True)
        await proxy_session_passthrough(client_ws, conn_id, first.data)

    return client_ws


# ── main ─────────────────────────────────────────────────────────────────────

async def main() -> None:
    global _internal_base_url, _internal_ws_url

    parser = argparse.ArgumentParser()
    parser.add_argument("--host",     default="0.0.0.0")
    parser.add_argument("--port",     type=int, default=8091)
    parser.add_argument("--upstream", default="ws://localhost:8095/v1/audio/speech/stream")
    parser.add_argument("--rest",     default=None,
                        help="Internal REST base URL (default: derived from --upstream)")
    args = parser.parse_args()

    _internal_ws_url = args.upstream
    if args.rest:
        _internal_base_url = args.rest.rstrip("/")
    else:
        http = args.upstream.replace("ws://", "http://").replace("wss://", "https://")
        _internal_base_url = http.rsplit("/v1/", 1)[0]

    print(f"Proxy on {args.host}:{args.port}  "
          f"(REST: {_internal_base_url}/v1/audio/speech  "
          f"WS: {_internal_ws_url})", flush=True)

    app = web.Application()
    app.router.add_get("/v1/audio/speech/stream", handle_ws)
    app.router.add_post("/v1/audio/speech",        handle_rest)
    app.router.add_get("/health",                  handle_health)

    runner = web.AppRunner(app, access_log=None)
    await runner.setup()
    site = web.TCPSite(runner, args.host, args.port)
    await site.start()
    print(f"Listening on http+ws://{args.host}:{args.port}", flush=True)
    await asyncio.Future()


if __name__ == "__main__":
    asyncio.run(main())