po/Dokumenten-Check
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Externe Dokument-Referenzen erkennen, herunterladen & mitbewerten

Browse Source 
Nach der Transkription eines Dokuments werden referenzierte externe
Dokumente (AGB, Verhaltenskodex) erkannt, sicher heruntergeladen und ihr
Text dem Hauptdokument fuer die "Summen"-Bewertung angehaengt (Option D).
Die Referenzen werden zusaetzlich als eigene, herunterladbare Zeilen
gespeichert.

- DB: dc_project_documents um parent_document_id, is_reference, source_url
  erweitert (DC_REFERENCES_ALTER.sql)
- ORDS: Dokumentenliste um die Felder erweitert; neue Endpunkte
  POST /documents/:id/references und DELETE /projects/:id/references;
  q-Quote-Parserfehler im ai-costs-Handler ('[]') behoben
- Backend: ReferenceDownloadService (SSRF-sicher via DNS-/Public-IP-
  Validierung, Redirect-Pruefung, Groessen-/Timeout-Limits, jsoup fuer HTML),
  EvaluationService.extractReferenceUrls (LLM-basierte URL-Erkennung),
  Orchestrierung (Discovery in Phase A, Referenz-Skip + combinedText in Phase B)
- Config: dc.references.* in application.properties

Verifiziert an Projekt 144: AGB-PDF/-HTML heruntergeladen, verknuepft,
"Summen"-Bewertung zitiert AGB-Klauseln aus dem Referenzdokument.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Philipp Ostmeyer
2026-06-22 12:45:55 +02:00
parent d00525c15f
commit c75f460ec5
91 changed files with 3949 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files

440
Scripts/ORDS REST Services.sql
View File

@@ -170,7 +170,10 @@ BEGIN
p.current_phase,
p.current_doc_id,
p.current_question_id,
p.estimated_completion_at
p.estimated_completion_at,
p.notify_ok,
p.notify_hinweis,
p.notify_abweichend
FROM dc_projects p
WHERE p.id = :project_id
]'
@@ -335,12 +338,67 @@ BEGIN
p_pattern => 'projects/:project_id/complete',
p_method => 'POST',
p_source_type => ORDS.SOURCE_TYPE_PLSQL,
p_comments => 'Setzt status=COMPLETED und erstellt Markdown-Bericht',
p_comments => 'Setzt status=COMPLETED, speichert Java-Markdown, sendet pro Dokument ein PDF per E-Mail',
p_source => q'[
DECLARE
v_status dc_projects.status%TYPE;
v_now DATE := SYSDATE;
v_status dc_projects.status%TYPE;
v_now DATE := SYSDATE;
v_report_md CLOB;
v_doc_pdfs JSON_ARRAY_T;
-- Konvertiert den BLOB-Request-Body (UTF-8 JSON) in einen CLOB
FUNCTION body_to_clob(p_blob IN BLOB) RETURN CLOB IS
v_clob CLOB;
v_dest INTEGER := 1;
v_src INTEGER := 1;
v_lang INTEGER := DBMS_LOB.DEFAULT_LANG_CTX;
v_warning INTEGER;
BEGIN
IF p_blob IS NULL OR DBMS_LOB.GETLENGTH(p_blob) = 0 THEN RETURN NULL; END IF;
DBMS_LOB.CREATETEMPORARY(v_clob, TRUE);
DBMS_LOB.CONVERTTOCLOB(v_clob, p_blob, DBMS_LOB.LOBMAXSIZE,
v_dest, v_src, NLS_CHARSET_ID('AL32UTF8'), v_lang, v_warning);
RETURN v_clob;
END body_to_clob;
-- Dekodiert einen Base64-CLOB in einen BLOB (chunk-weise, 24000 Zeichen pro Schritt)
FUNCTION base64_to_blob(p_b64 IN CLOB) RETURN BLOB IS
v_blob BLOB;
v_raw RAW(32767);
v_len INTEGER;
v_chunk INTEGER := 24000;
v_offset INTEGER := 1;
v_sub VARCHAR2(32767);
BEGIN
DBMS_LOB.CREATETEMPORARY(v_blob, TRUE);
v_len := DBMS_LOB.GETLENGTH(p_b64);
WHILE v_offset <= v_len LOOP
v_sub := DBMS_LOB.SUBSTR(p_b64, v_chunk, v_offset);
v_raw := UTL_ENCODE.BASE64_DECODE(UTL_RAW.CAST_TO_RAW(v_sub));
DBMS_LOB.WRITEAPPEND(v_blob, UTL_RAW.LENGTH(v_raw), v_raw);
v_offset := v_offset + v_chunk;
END LOOP;
RETURN v_blob;
END base64_to_blob;
BEGIN
-- JSON-Body parsen: report_markdown + document_pdfs (Array)
DECLARE
v_body_clob CLOB;
v_json JSON_OBJECT_T;
BEGIN
v_body_clob := body_to_clob(:body);
IF v_body_clob IS NOT NULL THEN
v_json := JSON_OBJECT_T.parse(v_body_clob);
v_report_md := v_json.get_clob('report_markdown');
BEGIN
v_doc_pdfs := v_json.get_array('document_pdfs');
EXCEPTION WHEN OTHERS THEN v_doc_pdfs := NULL; END;
DBMS_LOB.FREETEMPORARY(v_body_clob);
END IF;
EXCEPTION WHEN OTHERS THEN NULL;
END;
SELECT status
INTO v_status
FROM dc_projects
@@ -354,20 +412,114 @@ BEGIN
END IF;
UPDATE dc_projects
SET status = 'COMPLETED',
completed_at = v_now,
progress = 100,
current_phase = 'COMPLETED',
current_doc_id = NULL,
current_question_id = NULL,
SET status = 'COMPLETED',
completed_at = v_now,
progress = 100,
current_phase = 'COMPLETED',
current_doc_id = NULL,
current_question_id = NULL,
estimated_completion_at = NULL
WHERE id = :project_id;
-- Markdown-Bericht generieren (Fehler ignoriert, Projekt bleibt COMPLETED)
BEGIN
DC_UTILS_PKG.generate_report(:project_id);
EXCEPTION WHEN OTHERS THEN NULL;
END;
IF v_report_md IS NOT NULL AND DBMS_LOB.GETLENGTH(v_report_md) > 0 THEN
-- Kombinierten Markdown-Bericht speichern
UPDATE dc_projects SET report_markdown = v_report_md WHERE id = :project_id;
-- E-Mail mit je einem PDF-Anhang pro Dokument versenden
BEGIN
DECLARE
v_email dc_projects.notification_email%TYPE;
v_name dc_projects.name%TYPE;
v_done dc_projects.completed_at%TYPE;
v_mail_id NUMBER;
v_ok_cnt NUMBER := 0;
v_warn_cnt NUMBER := 0;
v_dev_cnt NUMBER := 0;
v_total NUMBER := 0;
C_OK CONSTANT VARCHAR2(10 CHAR) := UNISTR('\2705');
C_NOK CONSTANT VARCHAR2(10 CHAR) := UNISTR('\274C');
C_WARN CONSTANT VARCHAR2(10 CHAR) := UNISTR('\26A0\FE0F');
CRLF CONSTANT VARCHAR2(2) := CHR(13)||CHR(10);
v_body VARCHAR2(4000);
v_doc_obj JSON_OBJECT_T;
v_pdf_b64 CLOB;
v_pdf_blob BLOB;
v_fname VARCHAR2(500);
BEGIN
SELECT notification_email, name, completed_at
INTO v_email, v_name, v_done
FROM dc_projects WHERE id = :project_id;
IF v_email IS NULL OR TRIM(v_email) IS NULL THEN
NULL; -- keine Empfängeradresse → abbrechen
ELSE
SELECT COUNT(*),
SUM(CASE WHEN deviation = 1 THEN 1 ELSE 0 END),
SUM(CASE WHEN warning = 1 AND deviation = 0 THEN 1 ELSE 0 END)
INTO v_total, v_dev_cnt, v_warn_cnt
FROM dc_results WHERE project_id = :project_id;
v_ok_cnt := v_total - v_dev_cnt - v_warn_cnt;
v_body :=
'Sehr geehrte Damen und Herren,' || CRLF ||
CRLF ||
'die automatische Dokumentenprüfung für das Projekt' || CRLF ||
'„' || v_name || '"' || CRLF ||
'wurde am ' || TO_CHAR(NVL(v_done, SYSDATE), 'DD.MM.YYYY') ||
' erfolgreich abgeschlossen.' || CRLF ||
CRLF ||
'Zusammenfassung der Prüfergebnisse:' || CRLF ||
C_OK || ' OK: ' || v_ok_cnt || CRLF ||
C_WARN || ' Hinweise: ' || v_warn_cnt || CRLF ||
C_NOK || ' Abweichungen: ' || v_dev_cnt || CRLF ||
' Gesamt: ' || v_total || ' Prüfpunkte' || CRLF ||
CRLF ||
'Die Prüfberichte der einzelnen Dokumente finden Sie als PDF-Anhänge.' || CRLF ||
CRLF ||
'Mit freundlichen Grüßen' || CRLF ||
'Ihr Dokumenten-Check-System' || CRLF ||
'Frigosped Express GmbH';
v_mail_id := APEX_MAIL.SEND(
p_to => v_email,
p_from => DC_UTILS_PKG.c_mail_from,
p_subj => 'Prüfbericht: ' || v_name,
p_body => v_body
);
-- Einen PDF-Anhang pro Dokument hinzufügen
IF v_doc_pdfs IS NOT NULL AND v_doc_pdfs.get_size > 0 THEN
FOR i IN 0 .. v_doc_pdfs.get_size - 1 LOOP
BEGIN
v_doc_obj := TREAT(v_doc_pdfs.get(i) AS JSON_OBJECT_T);
v_fname := v_doc_obj.get_string('filename');
v_pdf_b64 := v_doc_obj.get_clob('pdf_base64');
IF v_pdf_b64 IS NOT NULL AND DBMS_LOB.GETLENGTH(v_pdf_b64) > 0 THEN
v_pdf_blob := base64_to_blob(v_pdf_b64);
APEX_MAIL.ADD_ATTACHMENT(
p_mail_id => v_mail_id,
p_attachment => v_pdf_blob,
p_filename => NVL(v_fname, 'Pruefbericht.pdf'),
p_mime_type => 'application/pdf'
);
DBMS_LOB.FREETEMPORARY(v_pdf_blob);
DBMS_LOB.FREETEMPORARY(v_pdf_b64);
END IF;
EXCEPTION WHEN OTHERS THEN NULL;
END;
END LOOP;
END IF;
END IF;
EXCEPTION WHEN OTHERS THEN NULL;
END;
END;
ELSE
-- Fallback: Bericht in DB generieren (z.B. bei leerem Body)
BEGIN
DC_UTILS_PKG.generate_report(:project_id);
EXCEPTION WHEN OTHERS THEN NULL;
END;
END IF;
:status_code := 200;
HTP.P('{"project_id":' || :project_id
@@ -446,6 +598,11 @@ BEGIN
d.document_type_id,
d.catalog_id,
d.uploaded_at,
d.status,
d.progress,
d.parent_document_id,
NVL(d.is_reference, 0) AS is_reference,
d.source_url,
CASE WHEN d.original_text IS NOT NULL THEN 1 ELSE 0 END AS has_original_text,
CASE WHEN d.translated_text IS NOT NULL THEN 1 ELSE 0 END AS has_translated_text,
d.row_version,
@@ -594,6 +751,67 @@ BEGIN
);
-- ===========================================================================
-- TEMPLATE 9c: documents/:doc_id/progress (PUT)
-- ===========================================================================
ORDS.DEFINE_TEMPLATE(
p_module_name => 'frigosped.dc',
p_pattern => 'documents/:doc_id/progress',
p_priority => 0,
p_comments => 'Dokument-Fortschritt und -Status schreiben'
);
-- 9c. PUT /api/dc/documents/:doc_id/progress
-- Body: {"progress":45,"status":"IN_PROGRESS"}
ORDS.DEFINE_HANDLER(
p_module_name => 'frigosped.dc',
p_pattern => 'documents/:doc_id/progress',
p_method => 'PUT',
p_source_type => ORDS.SOURCE_TYPE_PLSQL,
p_comments => 'Setzt progress (0-100) und/oder status des Dokuments',
p_source => q'[
DECLARE
v_body CLOB;
v_progress NUMBER;
v_status VARCHAR2(20);
v_rows INTEGER;
BEGIN
v_body := :body_text;
v_progress := TO_NUMBER(JSON_VALUE(v_body, '$.progress'));
v_status := JSON_VALUE(v_body, '$.status');
IF v_progress IS NOT NULL AND (v_progress < 0 OR v_progress > 100) THEN
:status_code := 400;
HTP.P('{"error":"progress muss zwischen 0 und 100 liegen"}');
RETURN;
END IF;
UPDATE dc_project_documents
SET progress = CASE WHEN JSON_EXISTS(v_body, '$.progress') THEN v_progress ELSE progress END,
status = CASE WHEN JSON_EXISTS(v_body, '$.status') THEN v_status ELSE status END
WHERE id = :doc_id;
v_rows := SQL%ROWCOUNT;
IF v_rows = 0 THEN
:status_code := 404;
HTP.P('{"error":"Dokument nicht gefunden","doc_id":' || :doc_id || '}');
RETURN;
END IF;
:status_code := 200;
HTP.P('{"doc_id":' || :doc_id || ',"progress":' || NVL(v_progress, -1)
|| ',"status":"' || NVL(v_status, '') || '"}');
EXCEPTION
WHEN OTHERS THEN
ROLLBACK;
:status_code := 500;
HTP.P('{"error":"' || REPLACE(SQLERRM, '"', '''') || '"}');
END;
]'
);
-- ===========================================================================
-- TEMPLATE 10+11: projects/:project_id/results
-- (POST = Einfuegen, DELETE = Alle loeschen)
@@ -1143,7 +1361,7 @@ BEGIN
|| ',"cost_eur":' || ROUND(NVL(v_month_eur, 0), 6)
|| ',"total_tokens":'|| NVL(v_month_tokens, 0)
|| ',"call_count":' || NVL(v_month_calls, 0)
|| '},"monthly":' || NVL(v_monthly_json, '[]')
|| '},"monthly":' || NVL(v_monthly_json, '[' || CHR(93))
|| '}'
);
@@ -1156,6 +1374,196 @@ BEGIN
);
-- ===========================================================================
-- TEMPLATE 18: documents/:parent_id/references (POST)
-- Legt eine heruntergeladene externe Referenz als neue dc_project_documents-
-- Zeile an (is_reference=1, verknuepft mit dem Originaldokument).
-- ===========================================================================
ORDS.DEFINE_TEMPLATE(
p_module_name => 'frigosped.dc',
p_pattern => 'documents/:parent_id/references',
p_priority => 0,
p_comments => 'Heruntergeladene externe Referenz zu einem Dokument speichern'
);
-- 18. POST /api/dc/documents/:parent_id/references
-- Body: {"source_url":"https://...","filename":"AGB.pdf",
-- "mime_type":"application/pdf","file_base64":"JVBERi0x..."}
ORDS.DEFINE_HANDLER(
p_module_name => 'frigosped.dc',
p_pattern => 'documents/:parent_id/references',
p_method => 'POST',
p_source_type => ORDS.SOURCE_TYPE_PLSQL,
p_comments => '201 Created mit neuer doc-id; legt is_reference=1-Zeile mit BLOB an',
p_source => q'[
DECLARE
v_body_clob CLOB;
v_json JSON_OBJECT_T;
v_source_url VARCHAR2(2000);
v_filename VARCHAR2(255);
v_mime VARCHAR2(100);
v_file_b64 CLOB;
v_blob BLOB;
v_project_id NUMBER;
v_doc_type_id NUMBER;
v_new_id NUMBER;
-- Wandelt den (ggf. grossen) Request-BLOB in einen CLOB
FUNCTION body_to_clob(p_body IN BLOB) RETURN CLOB IS
v_clob CLOB;
v_dest INTEGER := 1;
v_src INTEGER := 1;
v_lang INTEGER := 0;
v_warning INTEGER := 0;
BEGIN
IF p_body IS NULL OR DBMS_LOB.GETLENGTH(p_body) = 0 THEN
RETURN NULL;
END IF;
DBMS_LOB.CREATETEMPORARY(v_clob, TRUE);
DBMS_LOB.CONVERTTOCLOB(v_clob, p_body, DBMS_LOB.LOBMAXSIZE,
v_dest, v_src, NLS_CHARSET_ID('AL32UTF8'), v_lang, v_warning);
RETURN v_clob;
END body_to_clob;
-- Dekodiert einen Base64-CLOB in einen BLOB (chunk-weise)
FUNCTION base64_to_blob(p_b64 IN CLOB) RETURN BLOB IS
v_blob BLOB;
v_raw RAW(32767);
v_chunk INTEGER := 24000;
v_offset INTEGER := 1;
v_len INTEGER;
v_sub VARCHAR2(32767);
BEGIN
DBMS_LOB.CREATETEMPORARY(v_blob, TRUE);
v_len := DBMS_LOB.GETLENGTH(p_b64);
WHILE v_offset <= v_len LOOP
v_sub := DBMS_LOB.SUBSTR(p_b64, v_chunk, v_offset);
v_raw := UTL_ENCODE.BASE64_DECODE(UTL_RAW.CAST_TO_RAW(v_sub));
DBMS_LOB.WRITEAPPEND(v_blob, UTL_RAW.LENGTH(v_raw), v_raw);
v_offset := v_offset + v_chunk;
END LOOP;
RETURN v_blob;
END base64_to_blob;
BEGIN
v_body_clob := body_to_clob(:body);
IF v_body_clob IS NULL THEN
:status_code := 400;
HTP.P('{"error":"Leerer Request-Body"}');
RETURN;
END IF;
v_json := JSON_OBJECT_T.parse(v_body_clob);
v_source_url := v_json.get_string('source_url');
v_filename := v_json.get_string('filename');
v_mime := v_json.get_string('mime_type');
BEGIN
v_file_b64 := v_json.get_clob('file_base64');
EXCEPTION WHEN OTHERS THEN v_file_b64 := NULL; END;
DBMS_LOB.FREETEMPORARY(v_body_clob);
IF v_file_b64 IS NULL OR DBMS_LOB.GETLENGTH(v_file_b64) = 0 THEN
:status_code := 400;
HTP.P('{"error":"file_base64 ist Pflichtfeld"}');
RETURN;
END IF;
-- Projekt + Dokumenttyp vom Originaldokument uebernehmen
SELECT project_id, document_type_id
INTO v_project_id, v_doc_type_id
FROM dc_project_documents
WHERE id = :parent_id;
v_blob := base64_to_blob(v_file_b64);
INSERT INTO dc_project_documents (
project_id,
document_type_id,
parent_document_id,
is_reference,
source_url,
original_file,
mime_type,
filename,
status,
progress,
uploaded_at
) VALUES (
v_project_id,
v_doc_type_id,
:parent_id,
1,
v_source_url,
v_blob,
NVL(v_mime, 'application/octet-stream'),
NVL(v_filename, 'referenz'),
'IN_PROGRESS',
0,
SYSDATE
)
RETURNING id INTO v_new_id;
:status_code := 201;
HTP.P('{"id":' || v_new_id
|| ',"parent_document_id":' || :parent_id
|| ',"project_id":' || v_project_id || '}');
EXCEPTION
WHEN NO_DATA_FOUND THEN
ROLLBACK;
:status_code := 404;
HTP.P('{"error":"Originaldokument nicht gefunden","parent_id":' || :parent_id || '}');
WHEN OTHERS THEN
ROLLBACK;
:status_code := 500;
HTP.P('{"error":"' || REPLACE(SQLERRM, '"', '''') || '"}');
END;
]'
);
-- ===========================================================================
-- TEMPLATE 19: projects/:project_id/references (DELETE)
-- Loescht alle heruntergeladenen Referenzen eines Projekts (Re-Processing).
-- ===========================================================================
ORDS.DEFINE_TEMPLATE(
p_module_name => 'frigosped.dc',
p_pattern => 'projects/:project_id/references',
p_priority => 0,
p_comments => 'Heruntergeladene Referenzen eines Projekts verwalten'
);
-- 19. DELETE /api/dc/projects/:project_id/references
ORDS.DEFINE_HANDLER(
p_module_name => 'frigosped.dc',
p_pattern => 'projects/:project_id/references',
p_method => 'DELETE',
p_source_type => ORDS.SOURCE_TYPE_PLSQL,
p_comments => 'Loescht alle is_reference=1-Zeilen eines Projekts (fuer Re-Processing)',
p_source => q'[
DECLARE
v_deleted INTEGER;
BEGIN
DELETE FROM dc_project_documents
WHERE project_id = :project_id
AND is_reference = 1;
v_deleted := SQL%ROWCOUNT;
:status_code := 200;
HTP.P('{"project_id":' || :project_id
|| ',"deleted_count":' || v_deleted || '}');
EXCEPTION
WHEN OTHERS THEN
ROLLBACK;
:status_code := 500;
HTP.P('{"error":"' || REPLACE(SQLERRM, '"', '''') || '"}');
END;
]'
);
COMMIT;
END;
/