522bc21439
- Implemented MarkdownExportService to convert Markdown text into DOCX and PDF formats. - Supported Markdown elements include headings, bold, italic, blockquotes, tables, and specific emojis. - Added methods for generating DOCX and PDF documents with proper formatting and rendering. - Enhanced application properties to enable logging for production environment. - Introduced a test-run script to reset project state and trigger processing via API. Co-authored-by: Copilot <copilot@github.com>
395 lines
14 KiB
Markdown
395 lines
14 KiB
Markdown
# Dokumenten-Check
|
||
|
||
Projekt: https://projects.zoho.eu/portal/aquantico#project/323095000000333012
|
||
|
||
## Datenmodell
|
||
|
||
```mermaid
|
||
|
||
```
|
||
|
||
```mermaid
|
||
erDiagram
|
||
dc_users {
|
||
number id PK
|
||
varchar2_50 username UK
|
||
varchar2_100 email UK
|
||
varchar2_255 password_hash
|
||
varchar2_20 role
|
||
number is_active
|
||
date created_at
|
||
date updated_at
|
||
}
|
||
dc_user_roles {
|
||
number id PK
|
||
varchar2_20 role_name UK
|
||
clob description
|
||
}
|
||
dc_document_types {
|
||
number id PK
|
||
varchar2_100 name
|
||
clob description
|
||
}
|
||
dc_reference_documents {
|
||
number id PK
|
||
number uploaded_by FK
|
||
number document_type_id FK
|
||
varchar2_100 name
|
||
clob description
|
||
varchar2_500 file_path
|
||
date upload_date
|
||
varchar2_100 mime_type
|
||
}
|
||
dc_question_catalogs {
|
||
number id PK
|
||
number created_by_user FK
|
||
number based_on_reference FK
|
||
number document_type_id FK
|
||
varchar2_100 name
|
||
varchar2_4000 description
|
||
}
|
||
dc_question_categories {
|
||
number id PK
|
||
number catalog_id FK
|
||
varchar2_100 name
|
||
clob description
|
||
}
|
||
dc_questions {
|
||
number id PK
|
||
number category_id FK
|
||
clob question_text
|
||
varchar2_20 evaluation_type
|
||
number threshold
|
||
varchar2_20 result_handling
|
||
clob example_0_percent
|
||
clob example_100_percent
|
||
}
|
||
dc_projects {
|
||
number id PK
|
||
number catalog_id FK
|
||
number created_by_user FK
|
||
varchar2_100 name
|
||
clob description
|
||
varchar2_20 status
|
||
number progress
|
||
date completed_at
|
||
varchar2_100 notification_email
|
||
}
|
||
dc_project_documents {
|
||
number id PK
|
||
number project_id FK
|
||
number document_type_id FK
|
||
blob original_file
|
||
blob translated_file
|
||
varchar2_255 filename
|
||
varchar2_100 mime_type
|
||
date uploaded_at
|
||
}
|
||
dc_results {
|
||
number id PK
|
||
number project_id FK
|
||
number question_id FK
|
||
number doc_id FK
|
||
clob answer
|
||
number score
|
||
varchar2_20 result_type
|
||
number deviation
|
||
number warning
|
||
clob the_comment
|
||
date evaluated_at
|
||
}
|
||
dc_users ||--o{ dc_reference_documents : "uploads"
|
||
dc_users ||--o{ dc_question_catalogs : "creates"
|
||
dc_users ||--o{ dc_projects : "creates"
|
||
dc_document_types ||--o{ dc_reference_documents : "used_in"
|
||
dc_document_types ||--o{ dc_question_catalogs : "used_in"
|
||
dc_document_types ||--o{ dc_project_documents : "used_in"
|
||
dc_reference_documents ||--o{ dc_question_catalogs : "based_on"
|
||
dc_question_catalogs ||--o{ dc_question_categories : "contains"
|
||
dc_question_categories ||--o{ dc_questions : "contains"
|
||
dc_question_catalogs ||--o{ dc_projects : "uses"
|
||
dc_projects ||--o{ dc_project_documents : "contains"
|
||
dc_projects ||--o{ dc_results : "has"
|
||
dc_questions ||--o{ dc_results : "answered_in"
|
||
dc_project_documents ||--o{ dc_results : "evaluated_in"
|
||
```
|
||
|
||
## Backend
|
||
|
||
```mermaid
|
||
graph TB
|
||
subgraph Browser["🌐 Browser"]
|
||
APEX["Oracle APEX\nFrontend UI"]
|
||
end
|
||
|
||
subgraph OracleDB["🗄️ Oracle Database 23c"]
|
||
ORDS["ORDS REST Services\n/api/dc/..."]
|
||
subgraph Schema["DB Schema"]
|
||
Config["Konfiguration\ndc_document_types\ndc_question_catalogs\ndc_question_categories\ndc_questions"]
|
||
Exec["Ausführung\ndc_projects\ndc_project_documents\ndc_results"]
|
||
Users["dc_users\ndc_user_roles\ndc_reference_documents"]
|
||
end
|
||
ORDS <--> Schema
|
||
end
|
||
|
||
subgraph Backend["☕ Quarkus Backend (Port 8090)"]
|
||
CheckRes["CheckResource\nPOST /check/{projectId}"]
|
||
Orch["CheckOrchestrationService\nHauptablauf (async)"]
|
||
DocProc["DocumentProcessingService\nPDF/DOCX/ODT → Markdown"]
|
||
EvalSvc["EvaluationService\nÜbersetzung + Auswertung"]
|
||
OrdsClient["OrdsClient\nMicroProfile REST Client"]
|
||
|
||
CheckRes -->|"202 Accepted\nFire & Forget"| Orch
|
||
Orch --> DocProc
|
||
Orch --> EvalSvc
|
||
Orch --> OrdsClient
|
||
end
|
||
|
||
subgraph AI["🤖 Ollama (ollama.aquantico.de)"]
|
||
MainModel["gpt-oss:20b\nÜbersetzung + Fragenauswertung"]
|
||
OcrModel["quen3.5:9b\nPDF Vision-OCR"]
|
||
end
|
||
|
||
APEX -->|"Dokument hochladen\nProjekt anlegen"| ORDS
|
||
APEX -->|"POST /check/{id}"| CheckRes
|
||
APEX -->|"Ergebnisse lesen"| ORDS
|
||
|
||
OrdsClient -->|"Projekt/Dok/Fragen laden\nErgebnisse speichern"| ORDS
|
||
|
||
DocProc -->|"Base64-Bild\nX-API-KEY"| OcrModel
|
||
EvalSvc -->|"Text + Prompt\nX-API-KEY"| MainModel
|
||
|
||
subgraph PhaseA["Phase A (pro Dokument)"]
|
||
A1["Download BLOB"] --> A2["OCR / Konvertierung\n→ Markdown"]
|
||
A2 --> A3["Übersetzung → Deutsch"]
|
||
A3 --> A4["Texte speichern (ORDS)"]
|
||
end
|
||
|
||
subgraph PhaseB["Phase B (pro Dok × Frage)"]
|
||
B1["KI-Auswertung\nscore 0–100"] --> B2["Abweichung/Hinweis\nberechnen (Java)"]
|
||
B2 --> B3["Ergebnis speichern\n(ORDS)"]
|
||
end
|
||
|
||
Orch --> PhaseA
|
||
Orch --> PhaseB
|
||
|
||
style Browser fill:#dbeafe
|
||
style OracleDB fill:#fef3c7
|
||
style Backend fill:#dcfce7
|
||
style AI fill:#f3e8ff
|
||
style PhaseA fill:#f0f9ff
|
||
style PhaseB fill:#f0f9ff
|
||
```
|
||
|
||
|
||
|
||
|
||
|
||
## Dialoge
|
||
|
||
### Fragenkatalog
|
||
|
||
#### Tabelle
|
||
|
||
* Name
|
||
* Beschreibung
|
||
* Referenz-Dokument
|
||
|
||
#### Dialog
|
||
|
||
* Name (Im Kopf aussuchen)
|
||
* Referenz-Dokument Upload (Im Kopf)
|
||
* Master/Detail
|
||
* Fragen-Kategorien
|
||
* Fragen
|
||
|
||
|
||
|
||
### Projekte
|
||
|
||
#### Tabelle
|
||
|
||
* Name
|
||
* Beschreibung
|
||
* Anz ok,nok,undefiniert
|
||
|
||
#### Dialog
|
||
|
||
* Name (Im Kopf Aussuchen)
|
||
* Beschreibung
|
||
* Master/Details
|
||
* Dokumente
|
||
* Ergebnisse
|
||
|
||
|
||
|
||
## Source
|
||
|
||
```mermaid
|
||
sequenceDiagram
|
||
autonumber
|
||
|
||
participant Client as APEX / Client
|
||
participant AKF as ApiKeyFilter
|
||
participant CR as CheckResource
|
||
participant COS as CheckOrchestrationService
|
||
participant OLF as OrdsLoggingFilter
|
||
participant OC as OrdsClient
|
||
participant DPS as DocumentProcessingService
|
||
participant ES as EvaluationService
|
||
participant OCR_MDL as OllamaChatModel [ocr]
|
||
participant MAIN_MDL as OllamaChatModel [main]
|
||
participant ORDS as ORDS / Oracle DB
|
||
participant OLLAMA as ollama.aquantico.de
|
||
|
||
%% ─── Eingehende Anfrage ──────────────────────────────────────────────────
|
||
Client->>AKF: POST /check/{projectId} (Header: X-API-KEY)
|
||
AKF->>AKF: filter(ContainerRequestContext)
|
||
Note over AKF: Prüft X-API-KEY gegen dc.api.key<br/>→ 401 wenn ungültig, sonst weiter
|
||
AKF->>CR: weiterleiten (Key gültig)
|
||
|
||
CR->>CR: startCheck(projectId)
|
||
CR-->>Client: 202 Accepted { message, project_id, hint }
|
||
|
||
Note over CR,COS: CompletableFuture.runAsync() – Fire & Forget
|
||
CR-)COS: process(projectId)
|
||
|
||
%% ─── Alle ORDS-Aufrufe laufen durch OrdsLoggingFilter ───────────────────
|
||
Note over OLF,ORDS: OrdsLoggingFilter.filter() loggt jeden Request/Response<br/>des REST-Clients (ClientRequestFilter + ClientResponseFilter)
|
||
|
||
%% ─── Initialisierung ────────────────────────────────────────────────────
|
||
rect rgb(235, 245, 255)
|
||
Note over COS,ORDS: Initialisierung
|
||
|
||
COS->>OC: getProject(projectId)
|
||
OC->>OLF: filter(ClientRequestContext)
|
||
OLF->>ORDS: GET /api/dc/projects/{id}
|
||
ORDS-->>OLF: OrdsProject (JSON)
|
||
OLF->>OLF: filter(ClientRequestContext, ClientResponseContext)
|
||
OLF-->>COS: OrdsProject
|
||
|
||
Note over COS: Status-Prüfung: nur PENDING wird verarbeitet
|
||
|
||
COS->>OC: startProject(projectId)
|
||
OC->>ORDS: POST /api/dc/projects/{id}/start
|
||
ORDS-->>COS: 200 OK | 409 Conflict
|
||
|
||
COS->>OC: getDocuments(projectId)
|
||
OC->>ORDS: GET /api/dc/projects/{id}/documents/
|
||
ORDS-->>COS: OrdsDocumentList
|
||
|
||
COS->>OC: getQuestions(catalogId)
|
||
OC->>ORDS: GET /api/dc/catalogs/{id}/questions/
|
||
ORDS-->>COS: OrdsQuestionList
|
||
|
||
COS->>OC: deleteResults(projectId)
|
||
OC->>ORDS: DELETE /api/dc/projects/{id}/results
|
||
Note over ORDS: Alte Ergebnisse löschen → Re-Processing möglich
|
||
end
|
||
|
||
%% ─── Phase A: OCR + Übersetzung ─────────────────────────────────────────
|
||
rect rgb(220, 255, 225)
|
||
Note over COS,OLLAMA: Phase A – OCR + Übersetzung (pro Dokument)
|
||
|
||
loop für jedes OrdsDocument
|
||
|
||
COS->>OC: downloadFile(docId)
|
||
OC->>ORDS: GET /api/dc/documents/{id}/file
|
||
ORDS-->>COS: byte[] (BLOB)
|
||
|
||
COS->>DPS: extractText(fileBytes, mimeType, filename)
|
||
DPS->>DPS: resolveMimeType(mimeType, filename)
|
||
|
||
alt MIME = application/pdf
|
||
DPS->>DPS: extractFromPdf(fileBytes)
|
||
DPS->>DPS: Loader.loadPDF(fileBytes) → PDDocument
|
||
DPS->>DPS: new PDFRenderer(pdf)
|
||
|
||
loop für jede PDF-Seite
|
||
DPS->>DPS: renderer.renderImageWithDPI(i, 200f, RGB)
|
||
DPS->>DPS: ImageIO.write(image, "PNG", baos) → Base64
|
||
DPS->>DPS: ocrPage(base64Image, pageNum, totalPages)
|
||
DPS->>OCR_MDL: generate(List<ChatMessage>)
|
||
Note over DPS,OCR_MDL: UserMessage: ImageContent(PNG) + TextContent(OCR-Prompt)
|
||
OCR_MDL->>OLLAMA: POST /api/chat [Vision, X-API-KEY]
|
||
OLLAMA-->>OCR_MDL: Markdown-Seitentext
|
||
OCR_MDL-->>DPS: String (Seite i)
|
||
end
|
||
|
||
else MIME = application/vnd...wordprocessingml / msword
|
||
DPS->>DPS: extractFromDocx(fileBytes)
|
||
DPS->>DPS: new XWPFDocument(stream)
|
||
loop für jedes Body-Element
|
||
DPS->>DPS: paragraphToMarkdown(XWPFParagraph)
|
||
DPS->>DPS: tableToMarkdown(XWPFTable)
|
||
end
|
||
|
||
else MIME = application/vnd.oasis.opendocument.text
|
||
DPS->>DPS: extractFromOdt(fileBytes)
|
||
DPS->>DPS: OdfTextDocument.loadDocument(stream)
|
||
DPS->>DPS: getElementsByTagName("text:p") → Markdown
|
||
|
||
else text/plain | text/markdown
|
||
Note over DPS: direkt als UTF-8 String zurück
|
||
end
|
||
|
||
DPS-->>COS: originalText (Markdown)
|
||
|
||
COS->>ES: translate(originalText)
|
||
ES->>MAIN_MDL: generate(List<ChatMessage>)
|
||
Note over ES,MAIN_MDL: SystemMessage: Übersetzer-Prompt<br/>UserMessage: originalText
|
||
MAIN_MDL->>OLLAMA: POST /api/chat [X-API-KEY]
|
||
OLLAMA-->>MAIN_MDL: Übersetzung (Deutsch)
|
||
MAIN_MDL-->>ES: Response<AiMessage>
|
||
ES-->>COS: translatedText
|
||
|
||
COS->>OC: updateTexts(docId, TextsRequest(original, translated))
|
||
OC->>ORDS: PUT /api/dc/documents/{id}/texts
|
||
|
||
COS->>COS: pushProgress(projectId, step, total)
|
||
COS->>OC: updateProgress(projectId, ProgressRequest(pct))
|
||
OC->>ORDS: PUT /api/dc/projects/{id}/progress
|
||
end
|
||
end
|
||
|
||
%% ─── Phase B: Fragen auswerten ───────────────────────────────────────────
|
||
rect rgb(255, 245, 220)
|
||
Note over COS,OLLAMA: Phase B – Fragenauswertung (pro Dokument × Frage)
|
||
|
||
loop für jedes OrdsDocument × jede OrdsQuestion
|
||
|
||
COS->>ES: evaluate(question, originalText)
|
||
ES->>ES: buildEvaluationSystem()
|
||
ES->>ES: buildEvaluationUser(question, documentText)
|
||
Note over ES: Prompt: Dokument + Frage + Schwellwert<br/>+ Beispiele 0% / 100%
|
||
ES->>MAIN_MDL: generate(List<ChatMessage>)
|
||
MAIN_MDL->>OLLAMA: POST /api/chat [X-API-KEY]
|
||
OLLAMA-->>MAIN_MDL: JSON { answer, score, result_type, the_comment }
|
||
MAIN_MDL-->>ES: Response<AiMessage>
|
||
|
||
ES->>ES: parseEvaluationResult(rawText, question)
|
||
ES->>ES: extractJson(text)
|
||
Note over ES: Regex: ```json...``` oder erstes {...} im Text
|
||
ES->>ES: objectMapper.readValue(json, EvaluationResult)
|
||
ES->>ES: deriveResultType(score, threshold) [Fallback]
|
||
ES-->>COS: EvaluationResult { answer, score, resultType, theComment }
|
||
|
||
Note over COS: score < threshold?<br/>result_handling=ABWEICHUNG → deviation=1<br/>result_handling=HINWEIS → warning=1
|
||
|
||
COS->>OC: saveResult(projectId, ResultRequest)
|
||
OC->>ORDS: POST /api/dc/projects/{id}/results → 201 Created
|
||
|
||
COS->>COS: pushProgress(projectId, step, total)
|
||
COS->>OC: updateProgress(projectId, ProgressRequest(pct))
|
||
OC->>ORDS: PUT /api/dc/projects/{id}/progress
|
||
end
|
||
end
|
||
|
||
%% ─── Abschluss ───────────────────────────────────────────────────────────
|
||
COS->>OC: completeProject(projectId)
|
||
OC->>ORDS: POST /api/dc/projects/{id}/complete
|
||
Note over ORDS: status → COMPLETED, completed_at = SYSDATE
|
||
|
||
```
|
||
|