paperless_to_openwebui.py aktualisiert

Reviewed-on: #1

paperless_to_openwebui.py

@@ -38,6 +38,17 @@ MAX_TEXT_LENGTH = int(os.getenv("MAX_TEXT_LENGTH", "2000000"))
 
 KNOWLEDGE_ID    = None
 
+# ============================================================
+# CONFIG – Ergänzung für OCR-Fix
+# ============================================================
+LLM_BASE_URL       = os.environ.get("LLM_BASE_URL", "http://ollama:11434/v1")
+LLM_API_KEY        = os.environ.get("LLM_API_KEY", "unused")
+LLM_MODEL          = os.environ.get("LLM_MODEL", "gemma3:12b")
+OCR_FIX_ENABLED    = os.environ.get("OCR_FIX_ENABLED", "true").lower() == "true"
+SPACING_THRESHOLD  = float(os.environ.get("SPACING_THRESHOLD", "0.5"))
+MIN_SPACED_LINES   = int(os.environ.get("MIN_SPACED_LINES", "3"))
+LLM_CHUNK_SIZE     = int(os.environ.get("LLM_CHUNK_SIZE", "2000"))
+
 # ============================================================
 # LOGGING
 # ============================================================
@@ -58,6 +69,16 @@ openwebui_headers = {
     "Authorization": f"Bearer {OPENWEBUI_TOKEN}",
 }
 
+
+from openai import OpenAI
+
+# ============================================================
+# LLM CLIENT
+# ============================================================
+if OCR_FIX_ENABLED:
+    llm_client = OpenAI(base_url=LLM_BASE_URL, api_key=LLM_API_KEY)
+    log.info("OCR fix enabled, using model: %s", LLM_MODEL)
+
 # ============================================================
 # SPLITTER
 # ============================================================
@@ -294,7 +315,9 @@ def clean_ocr_text(text: str):
         ch for ch in text
         if ch == "\n" or ch == "\t" or ord(ch) >= 32
     )
-    text = re.sub(r"\s+", " ", text)
+    # Nur horizontale Whitespaces zusammenfassen, Zeilenumbrüche erhalten!
+    text = re.sub(r"[^\S\n]+", " ", text)       # ← geändert
+    text = re.sub(r"\n{3,}", "\n\n", text)      # max 2 Leerzeilen
     if len(text) > MAX_TEXT_LENGTH:
         log.warning("Truncating text (%s chars)", len(text))
         text = text[:MAX_TEXT_LENGTH]
@@ -544,6 +567,135 @@ def upload_to_openwebui(filename: str, content: str):
         except Exception:
             pass
 
+# ============================================================
+# OCR FIX VIA LLM
+# ============================================================
+OCR_FIX_SYSTEM_PROMPT = """Du bist ein OCR-Nachbearbeitungs-Assistent.
+Deine Aufgabe ist es, fehlerhaft erkannten Text zu korrigieren.
+
+Regeln:
+- Entferne falsche Leerzeichen in gesperrtem Text (z.B. "V e r t r a g" → "Vertrag")
+- Korrigiere offensichtliche OCR-Fehler (0/O, l/1, rn/m, etc.)
+- Behalte die ursprüngliche Struktur bei (Absätze, Zeilenumbrüche, Aufzählungen)
+- Erfinde KEINE Inhalte hinzu
+- Wenn der Text bereits korrekt ist, gib ihn unverändert zurück
+- Gib NUR den korrigierten Text zurück, keine Erklärungen"""
+
+
+# Typische OCR-Artefakte die in deutschen/englischen Texten nicht vorkommen
+OCR_ARTIFACT_CHARS = set("ÿšžřťňďľščžŕůúýáíéóôąęśćżźñ")
+# Noch aggressiver: Zeichen die fast nie in DE/EN-Dokumenten auftauchen
+OCR_SUSPECT_PATTERNS = [
+    "fi", "fl", "ff", "ffi", "ffl",  # Ligaturen die OCR manchmal erzeugt
+    "†", "‡",                      # Oft falsch erkannte Sonderzeichen
+    "\u00ad",                       # Soft-Hyphen (unsichtbar, aber stört Suche)
+    "¬",                           # Oft falscher Zeilenumbruch-Marker
+]
+
+
+def needs_ocr_fix(text: str) -> bool:
+    """Schneller Vorfilter: Hat der Text gesperrte Zeichen oder OCR-Artefakte?"""
+    if not text or len(text) < 50:
+        return False
+
+    # ── Check 1: Gesperrter Text (Leerzeichen zwischen Buchstaben) ──
+    lines = text.split('\n')
+    spaced_lines = 0
+    for line in lines:
+        stripped = line.strip()
+        if len(stripped) < 5:
+            continue
+        non_space = stripped.replace(' ', '')
+        if len(non_space) > 0:
+            ratio = stripped.count(' ') / len(non_space)
+            if ratio > SPACING_THRESHOLD:
+                spaced_lines += 1
+
+    if spaced_lines >= MIN_SPACED_LINES:
+        log.debug("  → Spacing-Artefakte erkannt (%d Zeilen)", spaced_lines)
+        return True
+
+    # ── Check 2: Fremdzeichen die in DE/EN nicht vorkommen ──
+    artifact_count = sum(1 for ch in text if ch in OCR_ARTIFACT_CHARS)
+    if artifact_count > 3:  # Mehr als 3 solcher Zeichen = verdächtig
+        log.debug("  → OCR-Artefakt-Zeichen erkannt (%d Stück)", artifact_count)
+        return True
+
+    # ── Check 3: Verdächtige Patterns ──
+    suspect_count = sum(text.count(p) for p in OCR_SUSPECT_PATTERNS)
+    if suspect_count > 5:
+        log.debug("  → Verdächtige OCR-Patterns erkannt (%d Stück)", suspect_count)
+        return True
+
+    return False
+
+
+def fix_text_with_llm(text: str) -> str:
+    """Schickt einen Textblock zur Korrektur ans LLM."""
+    try:
+        response = llm_client.chat.completions.create(
+            model=LLM_MODEL,
+            messages=[
+                {"role": "system", "content": OCR_FIX_SYSTEM_PROMPT},
+                {"role": "user", "content": f"Korrigiere folgenden OCR-Text:\n\n{text}"}
+            ],
+            temperature=0.0,
+        )
+        return response.choices[0].message.content
+    except Exception as e:
+        log.error("LLM OCR fix failed: %s", e)
+        return text  # Fallback: Originaltext
+
+
+def fix_ocr_chunked(content: str) -> str:
+    """Verarbeitet langen Text absatzweise durchs LLM."""
+    paragraphs = content.split('\n\n')
+    chunks = []
+    current_chunk = ""
+
+    for para in paragraphs:
+        if len(current_chunk) + len(para) > LLM_CHUNK_SIZE:
+            if current_chunk:
+                chunks.append(current_chunk)
+            current_chunk = para
+        else:
+            current_chunk += ("\n\n" + para if current_chunk else para)
+
+    if current_chunk:
+        chunks.append(current_chunk)
+
+    fixed_chunks = []
+    for i, chunk in enumerate(chunks):
+        log.debug("  LLM fix chunk %d/%d", i + 1, len(chunks))
+        fixed_chunks.append(fix_text_with_llm(chunk))
+        time.sleep(0.2)
+
+    return "\n\n".join(fixed_chunks)
+
+
+def update_paperless_content(doc_id: int, new_content: str) -> bool:
+    """Schreibt korrigierten Text zurück nach Paperless."""
+    try:
+        r = requests.patch(
+            f"{PAPERLESS_URL}/api/documents/{doc_id}/",
+            headers={**paperless_headers, "Content-Type": "application/json"},
+            json={"content": new_content},
+            timeout=120,
+        )
+        if r.ok:
+            log.info("Updated Paperless content for doc=%s", doc_id)
+            return True
+        else:
+            log.warning(
+                "Failed to update Paperless content doc=%s status=%s",
+                doc_id, r.status_code,
+            )
+            return False
+    except Exception as e:
+        log.error("Paperless content update failed doc=%s: %s", doc_id, e)
+        return False
+
+            
 # ============================================================
 # SYNC – parallelisiert
 # ============================================================
@@ -556,12 +708,28 @@ def process_single_doc(doc, i: int, total: int):
     with db_lock:
         existing = get_synced_document(doc_id)
 
-    text = download_document_text(doc_id)
+    text = doc.get("content", "")
     if not text.strip():
         log.warning("Skipping empty doc=%s", doc_id)
         return "skipped"
     
-    text         = clean_ocr_text(text)
+    text = clean_ocr_text(text)
+    
+    # ── OCR Fix ───────────────────────────────────────────────
+    if OCR_FIX_ENABLED and needs_ocr_fix(text):
+        log.info("OCR fix needed for doc=%s '%s'", doc_id, title)
+        fixed_text = fix_ocr_chunked(text)
+        if fixed_text and fixed_text != text:
+            log.info("OCR fix applied for doc=%s (delta: %+d chars)",
+                     doc_id, len(fixed_text) - len(text))
+            update_paperless_content(doc_id, fixed_text)
+            text = fixed_text
+        else:
+            log.debug("OCR fix: no changes for doc=%s", doc_id)
+    else:
+        log.debug("OCR fix: not needed for doc=%s", doc_id)
+    # ──────────────────────────────────────────────────────────
+    
     chunk_docs   = build_chunk_documents(doc, text)
     content_hash = hashlib.sha256(
         ("".join(chunk_docs)).encode("utf-8")
@@ -708,6 +876,75 @@ def wait_for_openwebui():
         log.info("Not ready yet, retrying in 10s...")
         time.sleep(10)
 
+def ocr_fix_only_pass():
+    """
+    Geht alle Paperless-Dokumente durch und korrigiert nur den OCR-Text.
+    Kein Upload nach OpenWebUI – das erledigt der nächste reguläre Sync.
+    """
+    log.info("=== OCR FIX ONLY MODE ===")
+    log.info("Fetching all documents from Paperless...")
+
+    docs = []
+    page = 1
+    while True:
+        url = f"{PAPERLESS_URL}/api/documents/?page_size=100&page={page}"
+        r = requests.get(url, headers=paperless_headers, timeout=120)
+        r.raise_for_status()
+        data = r.json()
+        results = data.get("results", [])
+        if not results:
+            break
+        docs.extend(results)
+        if not data.get("next"):
+            break
+        page += 1
+
+    total = len(docs)
+    log.info("Found %s documents to check.", total)
+
+    fixed = 0
+    skipped = 0
+    errors = 0
+
+    for i, doc in enumerate(docs):
+        doc_id = doc["id"]
+        title = doc.get("title", f"doc_{doc_id}")
+        content = doc.get("content", "")
+
+        if not content or len(content.strip()) < 50:
+            skipped += 1
+            continue
+
+        if not needs_ocr_fix(content):
+            skipped += 1
+            if (i + 1) % 100 == 0:
+                log.info("  Progress: %d/%d (fixed=%d, skipped=%d)",
+                         i + 1, total, fixed, skipped)
+            continue
+
+        log.info("OCR fix needed: #%s '%s'", doc_id, title)
+        try:
+            fixed_text = fix_ocr_chunked(content)
+            if fixed_text and fixed_text != content:
+                if update_paperless_content(doc_id, fixed_text):
+                    fixed += 1
+                    log.info("  ✓ Fixed #%s (delta: %+d chars)",
+                             doc_id, len(fixed_text) - len(content))
+                else:
+                    errors += 1
+            else:
+                skipped += 1
+                log.debug("  No changes for #%s", doc_id)
+        except Exception as e:
+            log.error("  ✗ Error fixing #%s: %s", doc_id, e)
+            errors += 1
+
+    log.info("=== OCR FIX COMPLETE ===")
+    log.info("  Total:   %d", total)
+    log.info("  Fixed:   %d", fixed)
+    log.info("  Skipped: %d", skipped)
+    log.info("  Errors:  %d", errors)
+
 # ============================================================
 # MAIN
 # ============================================================
@@ -728,14 +965,23 @@ def calc_wait_time(target_time_str):
     return (target - n).total_seconds()
 
 if __name__ == "__main__":
+    OCR_FIX_ONLY = os.environ.get("OCR_FIX_ONLY", "false").lower() == "true"
+
+    if OCR_FIX_ONLY:
+        # Nur OCR fixen, kein Sync nach OpenWebUI
+        init_db()
+        ocr_fix_only_pass()
+        sys.exit(0)
+
+    # Normaler Betrieb
     RUN_AT = os.getenv("SYNC_TIME", "04:00")
     print(f"Container started, task will run daily at {RUN_AT}", flush=True)
     while True:
         try:
-            main()
             wait_seconds = calc_wait_time(RUN_AT)
             print(f"Next run in {wait_seconds / 3600:.2f} hours.", flush=True)
             time.sleep(wait_seconds)
+            main()
         except KeyboardInterrupt:
             print("Container stopped manually", flush=True)
             sys.exit(0)