feat: enhance PDF text extraction and add maintenance features

- Refactor text extraction functions to extract text from all pages instead of just the first page. - Implement logging for better debugging during file operations. - Add a new maintenance page with options to rebuild the database and run consistency checks. - Update styles for maintenance actions and improve UI elements. - Remove obsolete rebuild database functionality from the main interface.
2026-11-11 00:22:12 +01:00 · 2026-10-13 22:01:36 +02:00
parent ab232a276f
commit 34271f3bfa
13 changed files with 237 additions and 67 deletions
@@ -1,6 +1,7 @@
 from __future__ import annotations

 import json
+import logging
 import os
 import re
 from datetime import datetime, timezone
@@ -27,6 +28,8 @@ from .models import (

 router = APIRouter(tags=["files"])

+logger = logging.getLogger('uvicorn.error')
+

 def _storage(request: Request) -> StorageBackend:
    storage = getattr(request.app.state, "storage", None)
@@ -129,6 +132,7 @@ async def list_files(
    }

    if db.count_files() == 0:
+        logger.debug("SQLite is empty, rebuilding from sidecar metadata")
        for key in sorted(storage.list_objects(prefix="")):
            if not key.endswith(".meta.json"):
                continue
@@ -177,6 +181,7 @@ async def get_file(
    record = db.get_by_id(file_id)

    if record is None:
+        logger.debug(f"File '{file_id}' not found in SQLite, searching storage sidecars")
        keys = storage.list_objects(prefix="")
        for key in keys:
            if not key.endswith(".meta.json"):
@@ -193,8 +198,9 @@ async def get_file(

    sidecar_metadata: dict[str, Any] = {}
    if record.current_key:
+        metadata_key = f"{record.current_key.split('.')[0]}.meta.json" if record.current_key.endswith(".pdf") else f"{record.current_key}.meta.json"
        sidecar_metadata = _parse_file_metadata(
-            storage.get_object(f"{record.current_key}.meta.json")
+            storage.get_object(metadata_key)
        )
    if not sidecar_metadata or sidecar_metadata.get("sha256") not in {
        record.sha256,
@@ -257,8 +263,19 @@ async def get_file_pdf(
 ) -> Response:
    """Stream a PDF by key from storage when present."""
    storage = _storage(request)
-    keys = storage.list_objects(prefix="")
+    db = _db(request)
+    record = db.get_by_id(file_id)

+    key = record.current_key if record else None
+    if key and storage.object_exists(key):
+        payload = storage.get_object(key)
+        if payload is not None:
+            logger.debug(f"Streaming PDF for file_id={file_id} from key={key}")
+            return Response(content=payload, media_type="application/pdf")
+        raise HTTPException(status_code=404, detail="PDF bytes not found for the file")
+
+    logger.debug(f"PDF for file_id={file_id} not found in SQLite, searching storage sidecars")
+    keys = storage.list_objects(prefix="")
    for key in keys:
        if key.endswith(".meta.json"):
            metadata = _parse_file_metadata(storage.get_object(key))
@@ -476,6 +493,7 @@ async def validate_file(
    record = db.get_by_id(file_id)

    if record is None:
+        logger.debug(f"File '{file_id}' not found in SQLite, searching storage sidecars")
        keys = storage.list_objects(prefix="")
        for key in keys:
            if key.endswith(".meta.json"):
@@ -277,10 +277,10 @@ class SQLiteFileRepository:
                params.append(f"%{confidence_filter.lower()}%")
        if q:
            conditions.append(
-                "(LOWER(current_key) LIKE ? OR LOWER(original_key) LIKE ? OR LOWER(template_name) LIKE ?  OR LOWER(document_type) LIKE ? OR LOWER(emitting_company) LIKE ? OR LOWER(period) LIKE ? OR LOWER(emission_date) LIKE ? OR LOWER(period_format) LIKE ?)"
+                "(LOWER(current_key) LIKE ? OR LOWER(original_key) LIKE ? OR LOWER(template_name) LIKE ? OR LOWER(document_type) LIKE ? OR LOWER(emitting_company) LIKE ? OR LOWER(period) LIKE ? OR LOWER(emission_date) LIKE ? OR LOWER(period_format) LIKE ? OR LOWER(extracted_text) LIKE ?)"
            )
            like_value = f"%{q.lower()}%"
-            params.extend([like_value] * 8)
+            params.extend([like_value] * 9)

        return conditions, params

@@ -19,17 +19,16 @@ class TextExtractionError(Exception):
    pass


-def extract_first_page_text(pdf_bytes: bytes) -> str:
-    """Extract text from the first page of a PDF.
+def extract_document_text(pdf_bytes: bytes) -> str:
+    """Extract text from every page of a PDF.

    Uses PyMuPDF for fast text extraction (PRD stack decision).
-    Extracts only the first page as per FR2.2.

    Args:
        pdf_bytes: The PDF file content as bytes.

    Returns:
-        The extracted text from the first page.
+        The extracted text from the complete document, in page order.

    Raises:
        TextExtractionError: If extraction fails or PDF is invalid.
@@ -42,33 +41,34 @@ def extract_first_page_text(pdf_bytes: bytes) -> str:
        )

    try:
-        # Open PDF from bytes
        doc = fitz.open(stream=pdf_bytes)
+        try:
+            if len(doc) == 0:
+                logger.warning("PDF has no pages")
+                return ""

-        if len(doc) == 0:
-            logger.warning("PDF has no pages")
-            return ""
-
-        # Extract text from first page only (FR2.2)
-        first_page = doc[0]
-        text = first_page.get_text()
-
-        doc.close()
-        return text
+            return "\n".join(page.get_text() for page in doc)
+        finally:
+            doc.close()

    except Exception as e:
        logger.error(f"Failed to extract text from PDF: {e}")
        raise TextExtractionError(f"Text extraction failed: {e}") from e


-def extract_first_page_text_from_file(filepath: str) -> str:
-    """Extract text from the first page of a PDF file.
+def extract_first_page_text(pdf_bytes: bytes) -> str:
+    """Compatibility wrapper for extract_document_text."""
+    return extract_document_text(pdf_bytes)
+
+
+def extract_document_text_from_file(filepath: str) -> str:
+    """Extract text from every page of a PDF file.

    Args:
        filepath: Path to the PDF file.

    Returns:
-        The extracted text from the first page.
+        The extracted text from the complete document.

    Raises:
        TextExtractionError: If extraction fails or PDF is invalid.
@@ -80,7 +80,12 @@ def extract_first_page_text_from_file(filepath: str) -> str:
    except FileNotFoundError:
        raise FileNotFoundError(f"PDF file not found: {filepath}")

-    return extract_first_page_text(pdf_bytes)
+    return extract_document_text(pdf_bytes)
+
+
+def extract_first_page_text_from_file(filepath: str) -> str:
+    """Compatibility wrapper for extract_document_text_from_file."""
+    return extract_document_text_from_file(filepath)


 def is_text_empty(text: str, min_length: int = MIN_TEXT_LENGTH) -> bool:
@@ -108,7 +113,7 @@ def is_text_empty(text: str, min_length: int = MIN_TEXT_LENGTH) -> bool:


 def extract_and_validate(pdf_bytes: bytes, min_length: int = MIN_TEXT_LENGTH) -> tuple[str, bool]:
-    """Extract first page text and check if it's valid.
+    """Extract document text and check if it's valid.

    Args:
        pdf_bytes: The PDF file content as bytes.
@@ -121,7 +126,7 @@ def extract_and_validate(pdf_bytes: bytes, min_length: int = MIN_TEXT_LENGTH) ->
    Raises:
        TextExtractionError: If extraction fails.
    """
-    text = extract_first_page_text(pdf_bytes)
+    text = extract_document_text(pdf_bytes)
    is_valid = not is_text_empty(text, min_length)
    return (text, is_valid)

@@ -142,7 +147,7 @@ class TextExtractor:
        self.min_length = min_length

    def extract(self, pdf_bytes: bytes) -> str:
-        """Extract first page text from PDF bytes.
+        """Extract text from every page of a PDF.

        Args:
            pdf_bytes: The PDF file content as bytes.
@@ -150,7 +155,7 @@ class TextExtractor:
        Returns:
            The extracted text.
        """
-        return extract_first_page_text(pdf_bytes)
+        return extract_document_text(pdf_bytes)

    def extract_and_validate(self, pdf_bytes: bytes) -> tuple[str, bool]:
        """Extract and validate text.
@@ -24,7 +24,7 @@ from .config import (
    MappingRule,
    PeriodFormatConfig,
 )
-from .extractor import extract_first_page_text, is_text_empty, TextExtractionError
+from .extractor import extract_document_text, is_text_empty, TextExtractionError
 from .period import (
    format_period_from_emission_date,
 )
@@ -99,7 +99,7 @@ class DocumentProcessor:
        result = ProcessingResult(success=False)

        try:
-            # Step 1: Extract first page text
+            # Step 1: Extract document text
            if extracted_text is None:
                extracted_text = self._extract_text(pdf_bytes)
            
@@ -293,7 +293,7 @@ class DocumentProcessor:
            Extracted text, or None if extraction fails.
        """
        try:
-            text = extract_first_page_text(pdf_bytes)
+            text = extract_document_text(pdf_bytes)
            if is_text_empty(text, self.min_text_length):
                return None
            return text
@@ -37,7 +37,7 @@ import requests
 # Add parent directory to path for imports
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

-from app.extractor import extract_first_page_text
+from app.extractor import extract_document_text
 from app.sidecar import (
    Sidecar,
    SidecarStatus,
@@ -826,7 +826,7 @@ class Importer:

            # Update sidecar
            sidecar.sha256 = sha256
-            sidecar.extracted_text = extract_first_page_text(pdf_bytes)
+            sidecar.extracted_text = extract_document_text(pdf_bytes)

            # Write sidecar
            sidecar_key = get_sidecar_key(target_key)
@@ -2,8 +2,6 @@ const searchFilter = document.getElementById('search-filter');
 const body = document.getElementById('documents-body');
 const summaryCards = document.getElementById('summary-cards');
 const refreshButton = document.getElementById('refresh-btn');
-const rebuildButton = document.getElementById('rebuild-btn');
-const rebuildStatus = document.getElementById('rebuild-status');
 const listStatus = document.getElementById('list-status');
 const loadMoreButton = document.getElementById('load-more-btn');
 const loadSentinel = document.getElementById('load-sentinel');
@@ -183,32 +181,9 @@ const updateSortIndicators = () => {
  });
 };

-const rebuildDatabase = async () => {
-  if (!window.confirm('Rebuild the database from sidecar files? Existing database records will be replaced.')) return;
-
-  rebuildButton.disabled = true;
-  rebuildStatus.textContent = 'Rebuilding database from sidecar files…';
-  try {
-    const response = await fetch('/api/consistency/rebuild', { method: 'POST' });
-    if (!response.ok) {
-      throw new Error(`HTTP ${response.status}`);
-    }
-
-    const result = await response.json();
-    rebuildStatus.textContent = `Database rebuilt from ${result.count} sidecar file${result.count === 1 ? '' : 's'}.`;
-    await loadDocuments();
-  } catch (error) {
-    rebuildStatus.textContent = 'Unable to rebuild the database. Check the API server and storage backend.';
-    console.error(error);
-  } finally {
-    rebuildButton.disabled = false;
-  }
-};
-
 columnFilters.forEach((filter) => filter.addEventListener('input', loadDocuments));
 searchFilter.addEventListener('input', loadDocuments);
 refreshButton.addEventListener('click', loadDocuments);
-rebuildButton.addEventListener('click', rebuildDatabase);
 loadMoreButton.addEventListener('click', () => loadDocuments({ append: true }));
 sortButtons.forEach((button) => {
  button.addEventListener('click', () => {
@@ -15,6 +15,7 @@
        <nav class="nav">
          <a href="/">Documents</a>
          <a class="active" href="/config.html">Config</a>
+          <a href="/maintenance.html">Maintenance</a>
        </nav>
      </aside>

@@ -15,6 +15,7 @@
        <nav class="nav">
          <a class="active" href="/">Documents</a>
          <a href="/config.html">Config</a>
+          <a href="/maintenance.html">Maintenance</a>
        </nav>
      </aside>

@@ -159,15 +159,25 @@ const buildAdditionalFieldRow = (label = '', value = '', suggestions = {}) => {
  const row = document.createElement('div');
  row.className = 'additional-field-row';

+  const labelGroup = document.createElement('div');
+  labelGroup.className = 'field-group';
+  const labelCaption = document.createElement('label');
+  labelCaption.textContent = 'Field name';
  const labelSelect = document.createElement('select');
  labelSelect.className = 'additional-field-label';
  labelSelect.setAttribute('aria-label', 'Additional field name');
+  labelGroup.append(labelCaption, labelSelect);
  const labelValues = [...new Set([...(suggestions.names || []), label].filter((item) => item !== null && item !== undefined && String(item).trim() !== ''))];
  setSelectOptions(labelSelect, labelValues, label);

+  const valueGroup = document.createElement('div');
+  valueGroup.className = 'field-group';
+  const valueCaption = document.createElement('label');
+  valueCaption.textContent = 'Value';
  const valueSelect = document.createElement('select');
  valueSelect.className = 'additional-field-value';
  valueSelect.setAttribute('aria-label', 'Additional field value');
+  valueGroup.append(valueCaption, valueSelect);
  const valueValues = [...new Set([...getFieldValueSuggestions(suggestions, labelSelect.value), value].filter((item) => item !== null && item !== undefined && String(item).trim() !== ''))];
  setSelectOptions(valueSelect, valueValues, value);
  labelSelect.addEventListener('change', () => {
@@ -189,7 +199,7 @@ const buildAdditionalFieldRow = (label = '', value = '', suggestions = {}) => {
    updateSaveButtonState();
  });

-  row.append(labelSelect, valueSelect, removeButton);
+  row.append(labelGroup, valueGroup, removeButton);
  return row;
 };

@@ -15,6 +15,7 @@
        <nav class="nav">
          <a class="active" href="/">Documents</a>
          <a href="/config.html">Config</a>
+          <a href="/maintenance.html">Maintenance</a>
        </nav>
      </aside>

@@ -26,18 +27,15 @@
          </div>
          <div class="toolbar-actions">
            <button id="refresh-btn" class="primary-btn">Refresh</button>
-            <button id="rebuild-btn" class="secondary-btn" type="button">Rebuild database</button>
          </div>
        </header>

-        <p id="rebuild-status" class="action-status" role="status" aria-live="polite"></p>
-
        <section class="summary-cards" id="summary-cards"></section>

        <section class="filters-card">
          <div class="field-group">
-            <label for="search-filter">Search all fields</label>
-            <input id="search-filter" type="text" placeholder="Search documents..." />
+            <label for="search-filter">Global search</label>
+            <input id="search-filter" type="text" placeholder="Search metadata and document content..." />
          </div>
        </section>

@@ -0,0 +1,60 @@
+<!DOCTYPE html>
+<html lang="fr">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Maintenance | Automatic PDF Renamer</title>
+    <link rel="stylesheet" href="/styles.css" />
+  </head>
+  <body>
+    <div class="app-shell">
+      <aside class="sidebar">
+        <div class="brand">
+          <h1>PDF Renamer</h1>
+        </div>
+        <nav class="nav">
+          <a href="/">Documents</a>
+          <a href="/config.html">Config</a>
+          <a class="active" href="/maintenance.html">Maintenance</a>
+        </nav>
+      </aside>
+
+      <main class="main-panel">
+        <header class="toolbar">
+          <div>
+            <p class="eyebrow">System tools</p>
+            <h2>Maintenance</h2>
+          </div>
+        </header>
+
+        <section class="maintenance-actions" aria-label="Maintenance actions">
+          <article class="maintenance-action">
+            <div>
+              <h3>Rebuild database</h3>
+              <p>Replace the database index using the metadata sidecar files in storage.</p>
+            </div>
+            <button id="rebuild-button" class="secondary-btn" type="button">Rebuild database</button>
+            <div id="rebuild-operation" class="maintenance-operation" hidden>
+              <progress id="rebuild-progress" aria-label="Database rebuild progress"></progress>
+              <p id="rebuild-status" class="action-status" role="status" aria-live="polite"></p>
+            </div>
+          </article>
+
+          <article class="maintenance-action">
+            <div>
+              <h3>Consistency check</h3>
+              <p>Compare database records and metadata sidecars with the files present in storage.</p>
+            </div>
+            <button id="check-button" class="secondary-btn" type="button">Run consistency check</button>
+            <div id="check-operation" class="maintenance-operation" hidden>
+              <progress id="check-progress" aria-label="Consistency check progress"></progress>
+              <p id="check-status" class="action-status" role="status" aria-live="polite"></p>
+            </div>
+          </article>
+        </section>
+      </main>
+    </div>
+
+    <script src="/maintenance.js"></script>
+  </body>
+</html>
@@ -0,0 +1,49 @@
+const rebuildButton = document.getElementById('rebuild-button');
+const checkButton = document.getElementById('check-button');
+const actionButtons = [rebuildButton, checkButton];
+
+const runAction = async ({ operation, progress, status, endpoint, method, confirmation, getResult }) => {
+  if (!window.confirm(confirmation)) return;
+
+  actionButtons.forEach((actionButton) => { actionButton.disabled = true; });
+  operation.hidden = false;
+  progress.hidden = false;
+  progress.removeAttribute('value');
+  status.textContent = 'Running...';
+
+  try {
+    const response = await fetch(endpoint, { method });
+    if (!response.ok) throw new Error(`HTTP ${response.status}`);
+    const result = await response.json();
+    progress.value = 1;
+    status.textContent = getResult(result);
+  } catch (error) {
+    progress.hidden = true;
+    status.textContent = 'The action failed. Check the API server and storage backend.';
+    console.error(error);
+  } finally {
+    actionButtons.forEach((actionButton) => { actionButton.disabled = false; });
+  }
+};
+
+rebuildButton.addEventListener('click', () => runAction({
+  operation: document.getElementById('rebuild-operation'),
+  progress: document.getElementById('rebuild-progress'),
+  status: document.getElementById('rebuild-status'),
+  endpoint: '/api/consistency/rebuild',
+  method: 'POST',
+  confirmation: 'Rebuild the database from sidecar files? Existing database records will be replaced.',
+  getResult: (result) => `Database rebuilt from ${result.count} sidecar file${result.count === 1 ? '' : 's'}.`,
+}));
+
+checkButton.addEventListener('click', () => runAction({
+  operation: document.getElementById('check-operation'),
+  progress: document.getElementById('check-progress'),
+  status: document.getElementById('check-status'),
+  endpoint: '/api/consistency',
+  method: 'GET',
+  confirmation: 'Run a consistency check? This scans metadata and database records against storage.',
+  getResult: (anomalies) => anomalies.length
+    ? `Consistency check complete: ${anomalies.length} issue${anomalies.length === 1 ? '' : 's'} found.`
+    : 'Consistency check complete: no issues found.',
+}));
@@ -172,6 +172,50 @@ body {
  padding: 18px;
 }

+.maintenance-actions {
+  display: grid;
+  gap: 14px;
+  max-width: 900px;
+}
+
+.maintenance-action {
+  display: grid;
+  grid-template-columns: minmax(0, 1fr) max-content;
+  align-items: center;
+  gap: 12px 24px;
+  padding: 18px;
+  background: var(--panel);
+  border: 1px solid var(--border);
+  border-radius: 8px;
+}
+
+.maintenance-action h3 {
+  margin: 0 0 6px;
+  font-size: 1.05rem;
+}
+
+.maintenance-action > div:first-child p {
+  margin: 0;
+  color: var(--muted);
+  line-height: 1.5;
+}
+
+.maintenance-operation {
+  grid-column: 1 / -1;
+}
+
+.maintenance-operation progress {
+  display: block;
+  width: 100%;
+  height: 8px;
+  accent-color: var(--primary);
+}
+
+.maintenance-operation .action-status {
+  min-height: 0;
+  margin: 8px 0 0;
+}
+
 .config-description {
  margin: 0 0 14px;
  color: var(--muted);
@@ -521,13 +565,14 @@ body {

 .additional-field-row {
  display: grid;
-  grid-template-columns: minmax(160px, 1fr) minmax(180px, 1.5fr) auto;
-  gap: 8px;
-  align-items: center;
+  grid-template-columns: minmax(0, 1fr) minmax(0, 1.5fr) auto;
+  gap: 10px;
+  align-items: end;
 }

-.additional-field-row input {
+.additional-field-row select {
  width: 100%;
+  min-width: 0;
 }

 .additional-field-empty {
@@ -688,6 +733,14 @@ body {
    grid-template-columns: 1fr;
  }

+  .maintenance-action {
+    grid-template-columns: minmax(0, 1fr);
+  }
+
+  .maintenance-action > button {
+    justify-self: start;
+  }
+
  .main-panel {
    padding: 18px;
  }