paperless_to_openwebui.py aktualisiert

paperless_to_openwebui.py

@@ -315,7 +315,9 @@ def clean_ocr_text(text: str):
         ch for ch in text
         if ch == "\n" or ch == "\t" or ord(ch) >= 32
     )
-    text = re.sub(r"\s+", " ", text)
+    # Nur horizontale Whitespaces zusammenfassen, Zeilenumbrüche erhalten!
+    text = re.sub(r"[^\S\n]+", " ", text)       # ← geändert
+    text = re.sub(r"\n{3,}", "\n\n", text)      # max 2 Leerzeilen
     if len(text) > MAX_TEXT_LENGTH:
         log.warning("Truncating text (%s chars)", len(text))
         text = text[:MAX_TEXT_LENGTH]
@@ -580,14 +582,25 @@ Regeln:
 - Gib NUR den korrigierten Text zurück, keine Erklärungen"""
 
 
+# Typische OCR-Artefakte die in deutschen/englischen Texten nicht vorkommen
+OCR_ARTIFACT_CHARS = set("ÿšžřťňďľščžŕůúýáíéóôąęśćżźñ")
+# Noch aggressiver: Zeichen die fast nie in DE/EN-Dokumenten auftauchen
+OCR_SUSPECT_PATTERNS = [
+    "fi", "fl", "ff", "ffi", "ffl",  # Ligaturen die OCR manchmal erzeugt
+    "†", "‡",                      # Oft falsch erkannte Sonderzeichen
+    "\u00ad",                       # Soft-Hyphen (unsichtbar, aber stört Suche)
+    "¬",                           # Oft falscher Zeilenumbruch-Marker
+]
+
+
 def needs_ocr_fix(text: str) -> bool:
-    """Schneller Vorfilter: Hat der Text gesperrte Zeichen?"""
+    """Schneller Vorfilter: Hat der Text gesperrte Zeichen oder OCR-Artefakte?"""
     if not text or len(text) < 50:
         return False
 
+    # ── Check 1: Gesperrter Text (Leerzeichen zwischen Buchstaben) ──
     lines = text.split('\n')
     spaced_lines = 0
-
     for line in lines:
         stripped = line.strip()
         if len(stripped) < 5:
@@ -598,7 +611,23 @@ def needs_ocr_fix(text: str) -> bool:
             if ratio > SPACING_THRESHOLD:
                 spaced_lines += 1
 
-    return spaced_lines >= MIN_SPACED_LINES
+    if spaced_lines >= MIN_SPACED_LINES:
+        log.debug("  → Spacing-Artefakte erkannt (%d Zeilen)", spaced_lines)
+        return True
+
+    # ── Check 2: Fremdzeichen die in DE/EN nicht vorkommen ──
+    artifact_count = sum(1 for ch in text if ch in OCR_ARTIFACT_CHARS)
+    if artifact_count > 3:  # Mehr als 3 solcher Zeichen = verdächtig
+        log.debug("  → OCR-Artefakt-Zeichen erkannt (%d Stück)", artifact_count)
+        return True
+
+    # ── Check 3: Verdächtige Patterns ──
+    suspect_count = sum(text.count(p) for p in OCR_SUSPECT_PATTERNS)
+    if suspect_count > 5:
+        log.debug("  → Verdächtige OCR-Patterns erkannt (%d Stück)", suspect_count)
+        return True
+
+    return False
 
 
 def fix_text_with_llm(text: str) -> str:
@@ -679,12 +708,28 @@ def process_single_doc(doc, i: int, total: int):
     with db_lock:
         existing = get_synced_document(doc_id)
 
-    text = download_document_text(doc_id)
+    text = doc.get("content", "")
     if not text.strip():
         log.warning("Skipping empty doc=%s", doc_id)
         return "skipped"
-
-    text         = clean_ocr_text(text)
+    
+    text = clean_ocr_text(text)
+    
+    # ── OCR Fix ───────────────────────────────────────────────
+    if OCR_FIX_ENABLED and needs_ocr_fix(text):
+        log.info("OCR fix needed for doc=%s '%s'", doc_id, title)
+        fixed_text = fix_ocr_chunked(text)
+        if fixed_text and fixed_text != text:
+            log.info("OCR fix applied for doc=%s (delta: %+d chars)",
+                     doc_id, len(fixed_text) - len(text))
+            update_paperless_content(doc_id, fixed_text)
+            text = fixed_text
+        else:
+            log.debug("OCR fix: no changes for doc=%s", doc_id)
+    else:
+        log.debug("OCR fix: not needed for doc=%s", doc_id)
+    # ──────────────────────────────────────────────────────────
+    
     chunk_docs   = build_chunk_documents(doc, text)
     content_hash = hashlib.sha256(
         ("".join(chunk_docs)).encode("utf-8")
@@ -831,10 +876,79 @@ def wait_for_openwebui():
         log.info("Not ready yet, retrying in 10s...")
         time.sleep(10)
 
+def ocr_fix_only_pass():
+    """
+    Geht alle Paperless-Dokumente durch und korrigiert nur den OCR-Text.
+    Kein Upload nach OpenWebUI – das erledigt der nächste reguläre Sync.
+    """
+    log.info("=== OCR FIX ONLY MODE ===")
+    log.info("Fetching all documents from Paperless...")
+
+    docs = []
+    page = 1
+    while True:
+        url = f"{PAPERLESS_URL}/api/documents/?page_size=100&page={page}"
+        r = requests.get(url, headers=paperless_headers, timeout=120)
+        r.raise_for_status()
+        data = r.json()
+        results = data.get("results", [])
+        if not results:
+            break
+        docs.extend(results)
+        if not data.get("next"):
+            break
+        page += 1
+
+    total = len(docs)
+    log.info("Found %s documents to check.", total)
+
+    fixed = 0
+    skipped = 0
+    errors = 0
+
+    for i, doc in enumerate(docs):
+        doc_id = doc["id"]
+        title = doc.get("title", f"doc_{doc_id}")
+        content = doc.get("content", "")
+
+        if not content or len(content.strip()) < 50:
+            skipped += 1
+            continue
+
+        if not needs_ocr_fix(content):
+            skipped += 1
+            if (i + 1) % 100 == 0:
+                log.info("  Progress: %d/%d (fixed=%d, skipped=%d)",
+                         i + 1, total, fixed, skipped)
+            continue
+
+        log.info("OCR fix needed: #%s '%s'", doc_id, title)
+        try:
+            fixed_text = fix_ocr_chunked(content)
+            if fixed_text and fixed_text != content:
+                if update_paperless_content(doc_id, fixed_text):
+                    fixed += 1
+                    log.info("  ✓ Fixed #%s (delta: %+d chars)",
+                             doc_id, len(fixed_text) - len(content))
+                else:
+                    errors += 1
+            else:
+                skipped += 1
+                log.debug("  No changes for #%s", doc_id)
+        except Exception as e:
+            log.error("  ✗ Error fixing #%s: %s", doc_id, e)
+            errors += 1
+
+    log.info("=== OCR FIX COMPLETE ===")
+    log.info("  Total:   %d", total)
+    log.info("  Fixed:   %d", fixed)
+    log.info("  Skipped: %d", skipped)
+    log.info("  Errors:  %d", errors)
+
 # ============================================================
 # MAIN
 # ============================================================
-def main():
+def main():        
     global KNOWLEDGE_ID
     init_db()
     wait_for_openwebui()
@@ -851,14 +965,23 @@ def calc_wait_time(target_time_str):
     return (target - n).total_seconds()
 
 if __name__ == "__main__":
+    OCR_FIX_ONLY = os.environ.get("OCR_FIX_ONLY", "false").lower() == "true"
+
+    if OCR_FIX_ONLY:
+        # Nur OCR fixen, kein Sync nach OpenWebUI
+        init_db()
+        ocr_fix_only_pass()
+        sys.exit(0)
+
+    # Normaler Betrieb
     RUN_AT = os.getenv("SYNC_TIME", "04:00")
     print(f"Container started, task will run daily at {RUN_AT}", flush=True)
     while True:
         try:
-            main()
             wait_seconds = calc_wait_time(RUN_AT)
             print(f"Next run in {wait_seconds / 3600:.2f} hours.", flush=True)
             time.sleep(wait_seconds)
+            main()
         except KeyboardInterrupt:
             print("Container stopped manually", flush=True)
             sys.exit(0)