From b26fff0d0ff537fe94d79f41cb7c947c859c769b Mon Sep 17 00:00:00 2001 From: wb Date: Tue, 23 Jun 2026 14:42:36 +0200 Subject: [PATCH] feat(diarization-ui): multi-provider LLM support via OpenAI-compatible API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace Ollama-native API (/api/generate) with OpenAI-compatible /v1/chat/completions — works with Ollama, OpenAI, Groq, LM Studio, LocalAI, and any other OpenAI-compatible endpoint - New providers table: name, base_url, optional API key, default flag - Settings page: add/delete/set-default providers; rename LLM params from ollama_* to llm_* settings keys - Per-prompt: choose provider from dropdown, model load button fetches from that provider's /v1/models - … tags extracted for live thinking display (works with Ollama thinking models and DeepSeek-R1 style responses) - Existing OLLAMA_BASE_URL env var seeds the initial provider on first start for backward compat Co-Authored-By: Claude Sonnet 4.6 --- app.py | 401 +++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 287 insertions(+), 114 deletions(-) diff --git a/app.py b/app.py index 4362325..5514dd2 100644 --- a/app.py +++ b/app.py @@ -5,6 +5,7 @@ import email.mime.multipart import email.mime.text import json import os +import re import smtplib import sqlite3 import threading @@ -19,10 +20,10 @@ from fastapi import FastAPI, File, Form, HTTPException, UploadFile from fastapi.responses import HTMLResponse, PlainTextResponse, Response, JSONResponse, RedirectResponse API_BASE = os.getenv("API_BASE", "http://gx10.aquantico.lan:8093").rstrip("/") -OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "http://gx10.aquantico.lan:11434").rstrip("/") -OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "qwen3.5:9b") -OLLAMA_NUM_PREDICT = int(os.getenv("OLLAMA_NUM_PREDICT", "16384")) -OLLAMA_THINK = os.getenv("OLLAMA_THINK", "true").lower() in ("1", "true", "yes") +# Kept for backward-compat seeding of the initial provider on first start +_SEED_PROVIDER_URL = os.getenv("OLLAMA_BASE_URL", "http://gx10.aquantico.lan:11434").rstrip("/") +_SEED_MODEL = os.getenv("OLLAMA_MODEL", "qwen3.5:9b") +_SEED_NUM_PREDICT = int(os.getenv("OLLAMA_NUM_PREDICT", "16384")) DB_PATH = os.getenv("DB_PATH", "/data/ui.db") app = FastAPI(title="Diarization UI") @@ -156,6 +157,35 @@ def init_db(): except Exception: pass + try: + c.execute( + """ + CREATE TABLE IF NOT EXISTS providers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + base_url TEXT NOT NULL, + api_key TEXT, + is_default INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL + ) + """ + ) + except Exception: + pass + + try: + c.execute("ALTER TABLE prompts ADD COLUMN provider_id INTEGER REFERENCES providers(id)") + except Exception: + pass + + # Seed default provider from env if table is empty + count = c.execute("SELECT COUNT(*) FROM providers").fetchone()[0] + if count == 0: + c.execute( + "INSERT INTO providers(name, base_url, api_key, is_default, created_at) VALUES (?,?,?,1,?)", + ("Ollama (Standard)", _SEED_PROVIDER_URL, None, now_iso()), + ) + try: c.execute( """ @@ -203,6 +233,32 @@ def set_setting(key: str, value: str): c.execute("INSERT OR REPLACE INTO settings(key, value) VALUES (?,?)", (key, value)) +def _get_default_provider() -> dict | None: + with db() as c: + row = c.execute("SELECT * FROM providers WHERE is_default=1 LIMIT 1").fetchone() + if not row: + row = c.execute("SELECT * FROM providers ORDER BY id LIMIT 1").fetchone() + return dict(row) if row else None + + +def _get_provider(provider_id: int | None) -> dict | None: + if not provider_id: + return _get_default_provider() + with db() as c: + row = c.execute("SELECT * FROM providers WHERE id=?", (provider_id,)).fetchone() + return dict(row) if row else _get_default_provider() + + +def _split_thinking(text: str) -> tuple[str, str]: + """Split … prefix from the rest of the content.""" + if text.startswith(""): + end = text.find("") + if end == -1: + return text[7:], "" + return text[7:end], text[end + 8:].lstrip("\n") + return "", text + + def _send_email(to_addrs: list[str], subject: str, body_md: str, attachment_name: str): host = get_setting("smtp_host", "") if not host: @@ -480,68 +536,98 @@ def _process_analysis_job(job_id: int): if not (doc["content_md"] or "").strip(): raise RuntimeError("Dokument hat keinen Inhalt – bitte zuerst das Transkript prüfen") - user_extra = (j.get("user_prompt") or "").strip() - llm_prompt = ( - "Du bist ein präziser Assistent. Antworte auf Deutsch.\\n" - f"AUFTRAG:\\n{prm['prompt']}\\n" - + (f"\\nZUSATZINFOS:\\n{user_extra}\\n" if user_extra else "") - + f"\\nTEXT:\\n{doc['content_md']}\\n" - ) - use_default = int(prm["llm_use_default"] if prm["llm_use_default"] is not None else 1) if use_default: - model = get_setting("ollama_model", OLLAMA_MODEL) - think = get_setting("ollama_think", "true" if OLLAMA_THINK else "false") == "true" - num_ctx_setting = get_setting("ollama_num_ctx", "auto") - num_predict = int(get_setting("ollama_num_predict", str(OLLAMA_NUM_PREDICT))) - repeat_penalty = float(get_setting("ollama_repeat_penalty", "1.15")) - repeat_last_n = int(get_setting("ollama_repeat_last_n", "128")) + provider = _get_default_provider() + model = get_setting("llm_model", _SEED_MODEL) + num_ctx_setting = get_setting("llm_num_ctx", "auto") + num_predict = int(get_setting("llm_num_predict", str(_SEED_NUM_PREDICT))) + repeat_penalty = float(get_setting("llm_repeat_penalty", "1.15")) + repeat_last_n = int(get_setting("llm_repeat_last_n", "128")) else: - model = prm["llm_model"] or get_setting("ollama_model", OLLAMA_MODEL) - think = (prm["llm_think"] or get_setting("ollama_think", "true" if OLLAMA_THINK else "false")) == "true" - num_ctx_setting = prm["llm_num_ctx"] or get_setting("ollama_num_ctx", "auto") - num_predict = int(prm["llm_num_predict"] or get_setting("ollama_num_predict", str(OLLAMA_NUM_PREDICT))) - repeat_penalty = float(prm["llm_repeat_penalty"] or get_setting("ollama_repeat_penalty", "1.15")) - repeat_last_n = int(prm["llm_repeat_last_n"] or get_setting("ollama_repeat_last_n", "128")) - num_ctx = _estimate_num_ctx(llm_prompt) if num_ctx_setting == "auto" else int(num_ctx_setting) - _job_set(job_id, llm_prompt=f"[num_ctx={num_ctx}]\n\n{llm_prompt}") + provider = _get_provider(prm.get("provider_id")) + model = prm["llm_model"] or get_setting("llm_model", _SEED_MODEL) + num_ctx_setting = prm["llm_num_ctx"] or get_setting("llm_num_ctx", "auto") + num_predict = int(prm["llm_num_predict"] or get_setting("llm_num_predict", str(_SEED_NUM_PREDICT))) + repeat_penalty = float(prm["llm_repeat_penalty"] or get_setting("llm_repeat_penalty", "1.15")) + repeat_last_n = int(prm["llm_repeat_last_n"] or get_setting("llm_repeat_last_n", "128")) + + if not provider: + raise RuntimeError("Kein LLM-Anbieter konfiguriert") + + user_extra = (j.get("user_prompt") or "").strip() + system_msg = "Du bist ein präziser Assistent. Antworte auf Deutsch." + user_msg = ( + f"AUFTRAG:\n{prm['prompt']}\n" + + (f"\nZUSATZINFOS:\n{user_extra}\n" if user_extra else "") + + f"\nTEXT:\n{doc['content_md']}\n" + ) + + ctx_hint = f"[num_ctx=auto]" if num_ctx_setting == "auto" else f"[num_ctx={num_ctx_setting}]" + _job_set(job_id, llm_prompt=f"{ctx_hint} [Anbieter: {provider['name']} | Modell: {model}]\n\nSYSTEM:\n{system_msg}\n\nUSER:\n{user_msg}") + with _JOB_STREAM_LOCK: _JOB_STREAMS[job_id] = {"thinking": "", "response": ""} - r = requests.post( - f"{OLLAMA_BASE_URL}/api/generate", - json={"model": model, "prompt": llm_prompt, "stream": True, "think": think, "options": { - "num_ctx": num_ctx, + + headers = {"Content-Type": "application/json"} + if provider.get("api_key"): + headers["Authorization"] = f"Bearer {provider['api_key']}" + + req_body: dict = { + "model": model, + "messages": [ + {"role": "system", "content": system_msg}, + {"role": "user", "content": user_msg}, + ], + "stream": True, + "max_tokens": num_predict, + } + if num_ctx_setting != "auto": + req_body["options"] = { + "num_ctx": int(num_ctx_setting), "num_predict": num_predict, "repeat_penalty": repeat_penalty, "repeat_last_n": repeat_last_n, - }}, + } + + r = requests.post( + f"{provider['base_url'].rstrip('/')}/v1/chat/completions", + json=req_body, + headers=headers, stream=True, timeout=1200, ) r.raise_for_status() - acc_thinking = "" - acc_response = "" - ollama_final = {} + acc = "" chunk_count = 0 + finish_reason = None for line in r.iter_lines(): if not line: continue - chunk = json.loads(line) - acc_thinking += chunk.get("thinking") or "" - acc_response += chunk.get("response") or "" + line_str = line.decode() if isinstance(line, bytes) else line + if not line_str.startswith("data: "): + continue + data_str = line_str[6:] + if data_str.strip() == "[DONE]": + break + try: + chunk = json.loads(data_str) + except Exception: + continue + choice = (chunk.get("choices") or [{}])[0] + acc += choice.get("delta", {}).get("content", "") or "" + finish_reason = choice.get("finish_reason") or finish_reason + t_part, r_part = _split_thinking(acc) with _JOB_STREAM_LOCK: - _JOB_STREAMS[job_id] = {"thinking": acc_thinking, "response": acc_response} + _JOB_STREAMS[job_id] = {"thinking": t_part, "response": r_part} chunk_count += 1 if chunk_count % 100 == 0: j_check = _job_get(job_id) if not j_check or j_check["status"] == "cancelled": return - if chunk.get("done"): - ollama_final = chunk - break - answer = acc_response + acc_thinking, answer = _split_thinking(acc) j = _job_get(job_id) if not j or j["status"] == "cancelled": @@ -560,7 +646,7 @@ def _process_analysis_job(job_id: int): answer, doc["id"], prm["id"], - json.dumps({"ollama_response": ollama_final}, ensure_ascii=False), + json.dumps({"provider": provider["name"], "model": model, "finish_reason": finish_reason}, ensure_ascii=False), now_iso(), ), ) @@ -667,15 +753,22 @@ self.addEventListener('fetch', (event) => { @app.get("/healthz") def healthz(): - return {"ok": True, "api_base": API_BASE, "ollama_base_url": OLLAMA_BASE_URL, "ollama_model": OLLAMA_MODEL, "db_path": DB_PATH} + return {"ok": True, "api_base": API_BASE, "db_path": DB_PATH} @app.get("/settings/models") -def settings_models(): +def settings_models(provider_id: Optional[int] = None): + provider = _get_provider(provider_id) + if not provider: + return JSONResponse({"ok": False, "error": "Kein Anbieter konfiguriert", "models": []}) + headers = {} + if provider.get("api_key"): + headers["Authorization"] = f"Bearer {provider['api_key']}" try: - r = requests.get(f"{OLLAMA_BASE_URL}/api/tags", timeout=5) + r = requests.get(f"{provider['base_url'].rstrip('/')}/v1/models", headers=headers, timeout=5) r.raise_for_status() - names = sorted(m["name"] for m in r.json().get("models", [])) + data = r.json() + names = sorted(m["id"] for m in data.get("data", [])) return JSONResponse({"ok": True, "models": names}) except Exception as e: return JSONResponse({"ok": False, "error": str(e), "models": []}) @@ -683,12 +776,11 @@ def settings_models(): @app.get("/settings", response_class=HTMLResponse) def settings_page(msg: str = ""): - current_model = get_setting("ollama_model", OLLAMA_MODEL) - current_think = get_setting("ollama_think", "true" if OLLAMA_THINK else "false") == "true" - current_num_ctx = get_setting("ollama_num_ctx", "auto") - current_num_predict = get_setting("ollama_num_predict", str(OLLAMA_NUM_PREDICT)) - current_repeat_penalty = get_setting("ollama_repeat_penalty", "1.15") - current_repeat_last_n = get_setting("ollama_repeat_last_n", "128") + current_model = get_setting("llm_model", _SEED_MODEL) + current_num_ctx = get_setting("llm_num_ctx", "auto") + current_num_predict = get_setting("llm_num_predict", str(_SEED_NUM_PREDICT)) + current_repeat_penalty = get_setting("llm_repeat_penalty", "1.15") + current_repeat_last_n = get_setting("llm_repeat_last_n", "128") smtp_host = get_setting("smtp_host", "") smtp_port = get_setting("smtp_port", "587") @@ -699,8 +791,25 @@ def settings_page(msg: str = ""): smtp_tls_checked = "checked" if smtp_tls else "" with db() as c: + providers = c.execute("SELECT * FROM providers ORDER BY name").fetchall() email_addrs = c.execute("SELECT id, label, email FROM email_addresses ORDER BY label").fetchall() + provider_rows = "".join( + f""" + {p["name"]} + {p["base_url"]} + + {'Standard' if p["is_default"] else f'
'} + +
+ +
+ + """ + for p in providers + ) + email_addr_rows = "".join( f""" {a["label"]} @@ -715,7 +824,9 @@ def settings_page(msg: str = ""): for a in email_addrs ) - think_checked = "checked" if current_think else "" + provider_id_opts = "".join( + f"" for p in providers + ) ctx_options = [("auto", "Automatisch (empfohlen)"), ("4096", "4 096"), ("8192", "8 192"), ("16384", "16 384"), ("32768", "32 768"), ("65536", "65 536"), ("131072", "131 072")] @@ -729,16 +840,48 @@ def settings_page(msg: str = ""):

Einstellungen

{f"
{msg}
" if msg else ""} -
-
Ollama / LLM
+ +
+
LLM-Anbieter
+

OpenAI-kompatible Endpunkte (Ollama, OpenAI, Groq, LM Studio, …)

+ {"" + provider_rows + "
NameBase-URLAPI-KeyStandard
" if providers else "

Noch kein Anbieter konfiguriert.

"} +
+
+ + +
+
+ + +
+
+ + +
+
+
+ + +
+ +
+
+
+ +
+
Standard-LLM-Parameter
+

Gelten für alle Prompts ohne eigene Einstellungen. Anbieter = markierter Standard-Anbieter oben.

- +
- + + @@ -751,38 +894,28 @@ def settings_page(msg: str = ""):
- +
Automatisch wählt passend zur Promptgröße.
-
-
-
-
-
- - -
-
-
@@ -851,13 +984,18 @@ def settings_page(msg: str = ""): -
API_BASE{API_BASE}
OLLAMA_BASE_URL{OLLAMA_BASE_URL}
DB_PATH{DB_PATH}