feat: enhance PDF text extraction and add maintenance features

- Refactor text extraction functions to extract text from all pages instead of just the first page.
- Implement logging for better debugging during file operations.
- Add a new maintenance page with options to rebuild the database and run consistency checks.
- Update styles for maintenance actions and improve UI elements.
- Remove obsolete rebuild database functionality from the main interface.

automatic-pdf-renamer/app/api/routes.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import json
+import logging
 import os
 import re
 from datetime import datetime, timezone
@@ -27,6 +28,8 @@ from .models import (
 
 router = APIRouter(tags=["files"])
 
+logger = logging.getLogger('uvicorn.error')
+
 
 def _storage(request: Request) -> StorageBackend:
     storage = getattr(request.app.state, "storage", None)
@@ -129,6 +132,7 @@ async def list_files(
     }
 
     if db.count_files() == 0:
+        logger.debug("SQLite is empty, rebuilding from sidecar metadata")
         for key in sorted(storage.list_objects(prefix="")):
             if not key.endswith(".meta.json"):
                 continue
@@ -177,6 +181,7 @@ async def get_file(
     record = db.get_by_id(file_id)
 
     if record is None:
+        logger.debug(f"File '{file_id}' not found in SQLite, searching storage sidecars")
         keys = storage.list_objects(prefix="")
         for key in keys:
             if not key.endswith(".meta.json"):
@@ -193,8 +198,9 @@ async def get_file(
 
     sidecar_metadata: dict[str, Any] = {}
     if record.current_key:
+        metadata_key = f"{record.current_key.split('.')[0]}.meta.json" if record.current_key.endswith(".pdf") else f"{record.current_key}.meta.json"
         sidecar_metadata = _parse_file_metadata(
-            storage.get_object(f"{record.current_key}.meta.json")
+            storage.get_object(metadata_key)
         )
     if not sidecar_metadata or sidecar_metadata.get("sha256") not in {
         record.sha256,
@@ -257,8 +263,19 @@ async def get_file_pdf(
 ) -> Response:
     """Stream a PDF by key from storage when present."""
     storage = _storage(request)
-    keys = storage.list_objects(prefix="")
+    db = _db(request)
+    record = db.get_by_id(file_id)
 
+    key = record.current_key if record else None
+    if key and storage.object_exists(key):
+        payload = storage.get_object(key)
+        if payload is not None:
+            logger.debug(f"Streaming PDF for file_id={file_id} from key={key}")
+            return Response(content=payload, media_type="application/pdf")
+        raise HTTPException(status_code=404, detail="PDF bytes not found for the file")
+
+    logger.debug(f"PDF for file_id={file_id} not found in SQLite, searching storage sidecars")
+    keys = storage.list_objects(prefix="")
     for key in keys:
         if key.endswith(".meta.json"):
             metadata = _parse_file_metadata(storage.get_object(key))
@@ -476,6 +493,7 @@ async def validate_file(
     record = db.get_by_id(file_id)
 
     if record is None:
+        logger.debug(f"File '{file_id}' not found in SQLite, searching storage sidecars")
         keys = storage.list_objects(prefix="")
         for key in keys:
             if key.endswith(".meta.json"):

automatic-pdf-renamer/app/db.py

@@ -277,10 +277,10 @@ class SQLiteFileRepository:
                 params.append(f"%{confidence_filter.lower()}%")
         if q:
             conditions.append(
-                "(LOWER(current_key) LIKE ? OR LOWER(original_key) LIKE ? OR LOWER(template_name) LIKE ?  OR LOWER(document_type) LIKE ? OR LOWER(emitting_company) LIKE ? OR LOWER(period) LIKE ? OR LOWER(emission_date) LIKE ? OR LOWER(period_format) LIKE ?)"
+                "(LOWER(current_key) LIKE ? OR LOWER(original_key) LIKE ? OR LOWER(template_name) LIKE ? OR LOWER(document_type) LIKE ? OR LOWER(emitting_company) LIKE ? OR LOWER(period) LIKE ? OR LOWER(emission_date) LIKE ? OR LOWER(period_format) LIKE ? OR LOWER(extracted_text) LIKE ?)"
             )
             like_value = f"%{q.lower()}%"
-            params.extend([like_value] * 8)
+            params.extend([like_value] * 9)
 
         return conditions, params
 

automatic-pdf-renamer/app/extractor.py

@@ -19,17 +19,16 @@ class TextExtractionError(Exception):
     pass
 
 
-def extract_first_page_text(pdf_bytes: bytes) -> str:
+def extract_document_text(pdf_bytes: bytes) -> str:
-    """Extract text from the first page of a PDF.
+    """Extract text from every page of a PDF.
 
     Uses PyMuPDF for fast text extraction (PRD stack decision).
-    Extracts only the first page as per FR2.2.
 
     Args:
         pdf_bytes: The PDF file content as bytes.
 
     Returns:
-        The extracted text from the first page.
+        The extracted text from the complete document, in page order.
 
     Raises:
         TextExtractionError: If extraction fails or PDF is invalid.
@@ -42,33 +41,34 @@ def extract_first_page_text(pdf_bytes: bytes) -> str:
         )
 
     try:
-        # Open PDF from bytes
         doc = fitz.open(stream=pdf_bytes)
+        try:
+            if len(doc) == 0:
+                logger.warning("PDF has no pages")
+                return ""
 
-        if len(doc) == 0:
+            return "\n".join(page.get_text() for page in doc)
-            logger.warning("PDF has no pages")
+        finally:
-            return ""
+            doc.close()
-
-        # Extract text from first page only (FR2.2)
-        first_page = doc[0]
-        text = first_page.get_text()
-
-        doc.close()
-        return text
 
     except Exception as e:
         logger.error(f"Failed to extract text from PDF: {e}")
         raise TextExtractionError(f"Text extraction failed: {e}") from e
 
 
-def extract_first_page_text_from_file(filepath: str) -> str:
+def extract_first_page_text(pdf_bytes: bytes) -> str:
-    """Extract text from the first page of a PDF file.
+    """Compatibility wrapper for extract_document_text."""
+    return extract_document_text(pdf_bytes)
+
+
+def extract_document_text_from_file(filepath: str) -> str:
+    """Extract text from every page of a PDF file.
 
     Args:
         filepath: Path to the PDF file.
 
     Returns:
-        The extracted text from the first page.
+        The extracted text from the complete document.
 
     Raises:
         TextExtractionError: If extraction fails or PDF is invalid.
@@ -80,7 +80,12 @@ def extract_first_page_text_from_file(filepath: str) -> str:
     except FileNotFoundError:
         raise FileNotFoundError(f"PDF file not found: {filepath}")
 
-    return extract_first_page_text(pdf_bytes)
+    return extract_document_text(pdf_bytes)
+
+
+def extract_first_page_text_from_file(filepath: str) -> str:
+    """Compatibility wrapper for extract_document_text_from_file."""
+    return extract_document_text_from_file(filepath)
 
 
 def is_text_empty(text: str, min_length: int = MIN_TEXT_LENGTH) -> bool:
@@ -108,7 +113,7 @@ def is_text_empty(text: str, min_length: int = MIN_TEXT_LENGTH) -> bool:
 
 
 def extract_and_validate(pdf_bytes: bytes, min_length: int = MIN_TEXT_LENGTH) -> tuple[str, bool]:
-    """Extract first page text and check if it's valid.
+    """Extract document text and check if it's valid.
 
     Args:
         pdf_bytes: The PDF file content as bytes.
@@ -121,7 +126,7 @@ def extract_and_validate(pdf_bytes: bytes, min_length: int = MIN_TEXT_LENGTH) ->
     Raises:
         TextExtractionError: If extraction fails.
     """
-    text = extract_first_page_text(pdf_bytes)
+    text = extract_document_text(pdf_bytes)
     is_valid = not is_text_empty(text, min_length)
     return (text, is_valid)
 
@@ -142,7 +147,7 @@ class TextExtractor:
         self.min_length = min_length
 
     def extract(self, pdf_bytes: bytes) -> str:
-        """Extract first page text from PDF bytes.
+        """Extract text from every page of a PDF.
 
         Args:
             pdf_bytes: The PDF file content as bytes.
@@ -150,7 +155,7 @@ class TextExtractor:
         Returns:
             The extracted text.
         """
-        return extract_first_page_text(pdf_bytes)
+        return extract_document_text(pdf_bytes)
 
     def extract_and_validate(self, pdf_bytes: bytes) -> tuple[str, bool]:
         """Extract and validate text.

automatic-pdf-renamer/app/pipeline.py

@@ -24,7 +24,7 @@ from .config import (
     MappingRule,
     PeriodFormatConfig,
 )
-from .extractor import extract_first_page_text, is_text_empty, TextExtractionError
+from .extractor import extract_document_text, is_text_empty, TextExtractionError
 from .period import (
     format_period_from_emission_date,
 )
@@ -99,7 +99,7 @@ class DocumentProcessor:
         result = ProcessingResult(success=False)
 
         try:
-            # Step 1: Extract first page text
+            # Step 1: Extract document text
             if extracted_text is None:
                 extracted_text = self._extract_text(pdf_bytes)
             
@@ -293,7 +293,7 @@ class DocumentProcessor:
             Extracted text, or None if extraction fails.
         """
         try:
-            text = extract_first_page_text(pdf_bytes)
+            text = extract_document_text(pdf_bytes)
             if is_text_empty(text, self.min_text_length):
                 return None
             return text

automatic-pdf-renamer/scripts/import_paperless.py

@@ -37,7 +37,7 @@ import requests
 # Add parent directory to path for imports
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from app.extractor import extract_first_page_text
+from app.extractor import extract_document_text
 from app.sidecar import (
     Sidecar,
     SidecarStatus,
@@ -826,7 +826,7 @@ class Importer:
 
             # Update sidecar
             sidecar.sha256 = sha256
-            sidecar.extracted_text = extract_first_page_text(pdf_bytes)
+            sidecar.extracted_text = extract_document_text(pdf_bytes)
 
             # Write sidecar
             sidecar_key = get_sidecar_key(target_key)

automatic-pdf-renamer/web/app.js

@@ -2,8 +2,6 @@ const searchFilter = document.getElementById('search-filter');
 const body = document.getElementById('documents-body');
 const summaryCards = document.getElementById('summary-cards');
 const refreshButton = document.getElementById('refresh-btn');
-const rebuildButton = document.getElementById('rebuild-btn');
-const rebuildStatus = document.getElementById('rebuild-status');
 const listStatus = document.getElementById('list-status');
 const loadMoreButton = document.getElementById('load-more-btn');
 const loadSentinel = document.getElementById('load-sentinel');
@@ -183,32 +181,9 @@ const updateSortIndicators = () => {
   });
 };
 
-const rebuildDatabase = async () => {
-  if (!window.confirm('Rebuild the database from sidecar files? Existing database records will be replaced.')) return;
-
-  rebuildButton.disabled = true;
-  rebuildStatus.textContent = 'Rebuilding database from sidecar files…';
-  try {
-    const response = await fetch('/api/consistency/rebuild', { method: 'POST' });
-    if (!response.ok) {
-      throw new Error(`HTTP ${response.status}`);
-    }
-
-    const result = await response.json();
-    rebuildStatus.textContent = `Database rebuilt from ${result.count} sidecar file${result.count === 1 ? '' : 's'}.`;
-    await loadDocuments();
-  } catch (error) {
-    rebuildStatus.textContent = 'Unable to rebuild the database. Check the API server and storage backend.';
-    console.error(error);
-  } finally {
-    rebuildButton.disabled = false;
-  }
-};
-
 columnFilters.forEach((filter) => filter.addEventListener('input', loadDocuments));
 searchFilter.addEventListener('input', loadDocuments);
 refreshButton.addEventListener('click', loadDocuments);
-rebuildButton.addEventListener('click', rebuildDatabase);
 loadMoreButton.addEventListener('click', () => loadDocuments({ append: true }));
 sortButtons.forEach((button) => {
   button.addEventListener('click', () => {

automatic-pdf-renamer/web/config.html

@@ -15,6 +15,7 @@
         <nav class="nav">
           <a href="/">Documents</a>
           <a class="active" href="/config.html">Config</a>
+          <a href="/maintenance.html">Maintenance</a>
         </nav>
       </aside>
 

automatic-pdf-renamer/web/document.html

@@ -15,6 +15,7 @@
         <nav class="nav">
           <a class="active" href="/">Documents</a>
           <a href="/config.html">Config</a>
+          <a href="/maintenance.html">Maintenance</a>
         </nav>
       </aside>
 

automatic-pdf-renamer/web/document.js

@@ -159,15 +159,25 @@ const buildAdditionalFieldRow = (label = '', value = '', suggestions = {}) => {
   const row = document.createElement('div');
   row.className = 'additional-field-row';
 
+  const labelGroup = document.createElement('div');
+  labelGroup.className = 'field-group';
+  const labelCaption = document.createElement('label');
+  labelCaption.textContent = 'Field name';
   const labelSelect = document.createElement('select');
   labelSelect.className = 'additional-field-label';
   labelSelect.setAttribute('aria-label', 'Additional field name');
+  labelGroup.append(labelCaption, labelSelect);
   const labelValues = [...new Set([...(suggestions.names || []), label].filter((item) => item !== null && item !== undefined && String(item).trim() !== ''))];
   setSelectOptions(labelSelect, labelValues, label);
 
+  const valueGroup = document.createElement('div');
+  valueGroup.className = 'field-group';
+  const valueCaption = document.createElement('label');
+  valueCaption.textContent = 'Value';
   const valueSelect = document.createElement('select');
   valueSelect.className = 'additional-field-value';
   valueSelect.setAttribute('aria-label', 'Additional field value');
+  valueGroup.append(valueCaption, valueSelect);
   const valueValues = [...new Set([...getFieldValueSuggestions(suggestions, labelSelect.value), value].filter((item) => item !== null && item !== undefined && String(item).trim() !== ''))];
   setSelectOptions(valueSelect, valueValues, value);
   labelSelect.addEventListener('change', () => {
@@ -189,7 +199,7 @@ const buildAdditionalFieldRow = (label = '', value = '', suggestions = {}) => {
     updateSaveButtonState();
   });
 
-  row.append(labelSelect, valueSelect, removeButton);
+  row.append(labelGroup, valueGroup, removeButton);
   return row;
 };
 

automatic-pdf-renamer/web/index.html

@@ -15,6 +15,7 @@
         <nav class="nav">
           <a class="active" href="/">Documents</a>
           <a href="/config.html">Config</a>
+          <a href="/maintenance.html">Maintenance</a>
         </nav>
       </aside>
 
@@ -26,18 +27,15 @@
           </div>
           <div class="toolbar-actions">
             <button id="refresh-btn" class="primary-btn">Refresh</button>
-            <button id="rebuild-btn" class="secondary-btn" type="button">Rebuild database</button>
           </div>
         </header>
 
-        <p id="rebuild-status" class="action-status" role="status" aria-live="polite"></p>
-
         <section class="summary-cards" id="summary-cards"></section>
 
         <section class="filters-card">
           <div class="field-group">
-            <label for="search-filter">Search all fields</label>
+            <label for="search-filter">Global search</label>
-            <input id="search-filter" type="text" placeholder="Search documents..." />
+            <input id="search-filter" type="text" placeholder="Search metadata and document content..." />
           </div>
         </section>
 

automatic-pdf-renamer/web/maintenance.html

@@ -0,0 +1,60 @@
+<!DOCTYPE html>
+<html lang="fr">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Maintenance | Automatic PDF Renamer</title>
+    <link rel="stylesheet" href="/styles.css" />
+  </head>
+  <body>
+    <div class="app-shell">
+      <aside class="sidebar">
+        <div class="brand">
+          <h1>PDF Renamer</h1>
+        </div>
+        <nav class="nav">
+          <a href="/">Documents</a>
+          <a href="/config.html">Config</a>
+          <a class="active" href="/maintenance.html">Maintenance</a>
+        </nav>
+      </aside>
+
+      <main class="main-panel">
+        <header class="toolbar">
+          <div>
+            <p class="eyebrow">System tools</p>
+            <h2>Maintenance</h2>
+          </div>
+        </header>
+
+        <section class="maintenance-actions" aria-label="Maintenance actions">
+          <article class="maintenance-action">
+            <div>
+              <h3>Rebuild database</h3>
+              <p>Replace the database index using the metadata sidecar files in storage.</p>
+            </div>
+            <button id="rebuild-button" class="secondary-btn" type="button">Rebuild database</button>
+            <div id="rebuild-operation" class="maintenance-operation" hidden>
+              <progress id="rebuild-progress" aria-label="Database rebuild progress"></progress>
+              <p id="rebuild-status" class="action-status" role="status" aria-live="polite"></p>
+            </div>
+          </article>
+
+          <article class="maintenance-action">
+            <div>
+              <h3>Consistency check</h3>
+              <p>Compare database records and metadata sidecars with the files present in storage.</p>
+            </div>
+            <button id="check-button" class="secondary-btn" type="button">Run consistency check</button>
+            <div id="check-operation" class="maintenance-operation" hidden>
+              <progress id="check-progress" aria-label="Consistency check progress"></progress>
+              <p id="check-status" class="action-status" role="status" aria-live="polite"></p>
+            </div>
+          </article>
+        </section>
+      </main>
+    </div>
+
+    <script src="/maintenance.js"></script>
+  </body>
+</html>

automatic-pdf-renamer/web/maintenance.js

@@ -0,0 +1,49 @@
+const rebuildButton = document.getElementById('rebuild-button');
+const checkButton = document.getElementById('check-button');
+const actionButtons = [rebuildButton, checkButton];
+
+const runAction = async ({ operation, progress, status, endpoint, method, confirmation, getResult }) => {
+  if (!window.confirm(confirmation)) return;
+
+  actionButtons.forEach((actionButton) => { actionButton.disabled = true; });
+  operation.hidden = false;
+  progress.hidden = false;
+  progress.removeAttribute('value');
+  status.textContent = 'Running...';
+
+  try {
+    const response = await fetch(endpoint, { method });
+    if (!response.ok) throw new Error(`HTTP ${response.status}`);
+    const result = await response.json();
+    progress.value = 1;
+    status.textContent = getResult(result);
+  } catch (error) {
+    progress.hidden = true;
+    status.textContent = 'The action failed. Check the API server and storage backend.';
+    console.error(error);
+  } finally {
+    actionButtons.forEach((actionButton) => { actionButton.disabled = false; });
+  }
+};
+
+rebuildButton.addEventListener('click', () => runAction({
+  operation: document.getElementById('rebuild-operation'),
+  progress: document.getElementById('rebuild-progress'),
+  status: document.getElementById('rebuild-status'),
+  endpoint: '/api/consistency/rebuild',
+  method: 'POST',
+  confirmation: 'Rebuild the database from sidecar files? Existing database records will be replaced.',
+  getResult: (result) => `Database rebuilt from ${result.count} sidecar file${result.count === 1 ? '' : 's'}.`,
+}));
+
+checkButton.addEventListener('click', () => runAction({
+  operation: document.getElementById('check-operation'),
+  progress: document.getElementById('check-progress'),
+  status: document.getElementById('check-status'),
+  endpoint: '/api/consistency',
+  method: 'GET',
+  confirmation: 'Run a consistency check? This scans metadata and database records against storage.',
+  getResult: (anomalies) => anomalies.length
+    ? `Consistency check complete: ${anomalies.length} issue${anomalies.length === 1 ? '' : 's'} found.`
+    : 'Consistency check complete: no issues found.',
+}));

automatic-pdf-renamer/web/styles.css

@@ -172,6 +172,50 @@ body {
   padding: 18px;
 }
 
+.maintenance-actions {
+  display: grid;
+  gap: 14px;
+  max-width: 900px;
+}
+
+.maintenance-action {
+  display: grid;
+  grid-template-columns: minmax(0, 1fr) max-content;
+  align-items: center;
+  gap: 12px 24px;
+  padding: 18px;
+  background: var(--panel);
+  border: 1px solid var(--border);
+  border-radius: 8px;
+}
+
+.maintenance-action h3 {
+  margin: 0 0 6px;
+  font-size: 1.05rem;
+}
+
+.maintenance-action > div:first-child p {
+  margin: 0;
+  color: var(--muted);
+  line-height: 1.5;
+}
+
+.maintenance-operation {
+  grid-column: 1 / -1;
+}
+
+.maintenance-operation progress {
+  display: block;
+  width: 100%;
+  height: 8px;
+  accent-color: var(--primary);
+}
+
+.maintenance-operation .action-status {
+  min-height: 0;
+  margin: 8px 0 0;
+}
+
 .config-description {
   margin: 0 0 14px;
   color: var(--muted);
@@ -521,13 +565,14 @@ body {
 
 .additional-field-row {
   display: grid;
-  grid-template-columns: minmax(160px, 1fr) minmax(180px, 1.5fr) auto;
+  grid-template-columns: minmax(0, 1fr) minmax(0, 1.5fr) auto;
-  gap: 8px;
+  gap: 10px;
-  align-items: center;
+  align-items: end;
 }
 
-.additional-field-row input {
+.additional-field-row select {
   width: 100%;
+  min-width: 0;
 }
 
 .additional-field-empty {
@@ -688,6 +733,14 @@ body {
     grid-template-columns: 1fr;
   }
 
+  .maintenance-action {
+    grid-template-columns: minmax(0, 1fr);
+  }
+
+  .maintenance-action > button {
+    justify-self: start;
+  }
+
   .main-panel {
     padding: 18px;
   }