From 34271f3bfad3f93f9d774967d56c9b7859fe752e Mon Sep 17 00:00:00 2001 From: thomas Date: Tue, 13 Oct 2026 22:01:36 +0200 Subject: [PATCH] feat: enhance PDF text extraction and add maintenance features - Refactor text extraction functions to extract text from all pages instead of just the first page. - Implement logging for better debugging during file operations. - Add a new maintenance page with options to rebuild the database and run consistency checks. - Update styles for maintenance actions and improve UI elements. - Remove obsolete rebuild database functionality from the main interface. --- automatic-pdf-renamer/app/api/routes.py | 22 ++++++- automatic-pdf-renamer/app/db.py | 4 +- automatic-pdf-renamer/app/extractor.py | 51 +++++++++------- automatic-pdf-renamer/app/pipeline.py | 6 +- .../scripts/import_paperless.py | 4 +- automatic-pdf-renamer/web/app.js | 25 -------- automatic-pdf-renamer/web/config.html | 1 + automatic-pdf-renamer/web/document.html | 1 + automatic-pdf-renamer/web/document.js | 12 +++- automatic-pdf-renamer/web/index.html | 8 +-- automatic-pdf-renamer/web/maintenance.html | 60 ++++++++++++++++++ automatic-pdf-renamer/web/maintenance.js | 49 +++++++++++++++ automatic-pdf-renamer/web/styles.css | 61 +++++++++++++++++-- 13 files changed, 237 insertions(+), 67 deletions(-) create mode 100644 automatic-pdf-renamer/web/maintenance.html create mode 100644 automatic-pdf-renamer/web/maintenance.js diff --git a/automatic-pdf-renamer/app/api/routes.py b/automatic-pdf-renamer/app/api/routes.py index 47b05fe..201fc9a 100644 --- a/automatic-pdf-renamer/app/api/routes.py +++ b/automatic-pdf-renamer/app/api/routes.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import logging import os import re from datetime import datetime, timezone @@ -27,6 +28,8 @@ from .models import ( router = APIRouter(tags=["files"]) +logger = logging.getLogger('uvicorn.error') + def _storage(request: Request) -> StorageBackend: storage = getattr(request.app.state, "storage", None) @@ -129,6 +132,7 @@ async def list_files( } if db.count_files() == 0: + logger.debug("SQLite is empty, rebuilding from sidecar metadata") for key in sorted(storage.list_objects(prefix="")): if not key.endswith(".meta.json"): continue @@ -177,6 +181,7 @@ async def get_file( record = db.get_by_id(file_id) if record is None: + logger.debug(f"File '{file_id}' not found in SQLite, searching storage sidecars") keys = storage.list_objects(prefix="") for key in keys: if not key.endswith(".meta.json"): @@ -193,8 +198,9 @@ async def get_file( sidecar_metadata: dict[str, Any] = {} if record.current_key: + metadata_key = f"{record.current_key.split('.')[0]}.meta.json" if record.current_key.endswith(".pdf") else f"{record.current_key}.meta.json" sidecar_metadata = _parse_file_metadata( - storage.get_object(f"{record.current_key}.meta.json") + storage.get_object(metadata_key) ) if not sidecar_metadata or sidecar_metadata.get("sha256") not in { record.sha256, @@ -257,8 +263,19 @@ async def get_file_pdf( ) -> Response: """Stream a PDF by key from storage when present.""" storage = _storage(request) - keys = storage.list_objects(prefix="") + db = _db(request) + record = db.get_by_id(file_id) + key = record.current_key if record else None + if key and storage.object_exists(key): + payload = storage.get_object(key) + if payload is not None: + logger.debug(f"Streaming PDF for file_id={file_id} from key={key}") + return Response(content=payload, media_type="application/pdf") + raise HTTPException(status_code=404, detail="PDF bytes not found for the file") + + logger.debug(f"PDF for file_id={file_id} not found in SQLite, searching storage sidecars") + keys = storage.list_objects(prefix="") for key in keys: if key.endswith(".meta.json"): metadata = _parse_file_metadata(storage.get_object(key)) @@ -476,6 +493,7 @@ async def validate_file( record = db.get_by_id(file_id) if record is None: + logger.debug(f"File '{file_id}' not found in SQLite, searching storage sidecars") keys = storage.list_objects(prefix="") for key in keys: if key.endswith(".meta.json"): diff --git a/automatic-pdf-renamer/app/db.py b/automatic-pdf-renamer/app/db.py index 6bac280..71038bc 100644 --- a/automatic-pdf-renamer/app/db.py +++ b/automatic-pdf-renamer/app/db.py @@ -277,10 +277,10 @@ class SQLiteFileRepository: params.append(f"%{confidence_filter.lower()}%") if q: conditions.append( - "(LOWER(current_key) LIKE ? OR LOWER(original_key) LIKE ? OR LOWER(template_name) LIKE ? OR LOWER(document_type) LIKE ? OR LOWER(emitting_company) LIKE ? OR LOWER(period) LIKE ? OR LOWER(emission_date) LIKE ? OR LOWER(period_format) LIKE ?)" + "(LOWER(current_key) LIKE ? OR LOWER(original_key) LIKE ? OR LOWER(template_name) LIKE ? OR LOWER(document_type) LIKE ? OR LOWER(emitting_company) LIKE ? OR LOWER(period) LIKE ? OR LOWER(emission_date) LIKE ? OR LOWER(period_format) LIKE ? OR LOWER(extracted_text) LIKE ?)" ) like_value = f"%{q.lower()}%" - params.extend([like_value] * 8) + params.extend([like_value] * 9) return conditions, params diff --git a/automatic-pdf-renamer/app/extractor.py b/automatic-pdf-renamer/app/extractor.py index 76682af..75b05ca 100644 --- a/automatic-pdf-renamer/app/extractor.py +++ b/automatic-pdf-renamer/app/extractor.py @@ -19,17 +19,16 @@ class TextExtractionError(Exception): pass -def extract_first_page_text(pdf_bytes: bytes) -> str: - """Extract text from the first page of a PDF. +def extract_document_text(pdf_bytes: bytes) -> str: + """Extract text from every page of a PDF. Uses PyMuPDF for fast text extraction (PRD stack decision). - Extracts only the first page as per FR2.2. Args: pdf_bytes: The PDF file content as bytes. Returns: - The extracted text from the first page. + The extracted text from the complete document, in page order. Raises: TextExtractionError: If extraction fails or PDF is invalid. @@ -42,33 +41,34 @@ def extract_first_page_text(pdf_bytes: bytes) -> str: ) try: - # Open PDF from bytes doc = fitz.open(stream=pdf_bytes) + try: + if len(doc) == 0: + logger.warning("PDF has no pages") + return "" - if len(doc) == 0: - logger.warning("PDF has no pages") - return "" - - # Extract text from first page only (FR2.2) - first_page = doc[0] - text = first_page.get_text() - - doc.close() - return text + return "\n".join(page.get_text() for page in doc) + finally: + doc.close() except Exception as e: logger.error(f"Failed to extract text from PDF: {e}") raise TextExtractionError(f"Text extraction failed: {e}") from e -def extract_first_page_text_from_file(filepath: str) -> str: - """Extract text from the first page of a PDF file. +def extract_first_page_text(pdf_bytes: bytes) -> str: + """Compatibility wrapper for extract_document_text.""" + return extract_document_text(pdf_bytes) + + +def extract_document_text_from_file(filepath: str) -> str: + """Extract text from every page of a PDF file. Args: filepath: Path to the PDF file. Returns: - The extracted text from the first page. + The extracted text from the complete document. Raises: TextExtractionError: If extraction fails or PDF is invalid. @@ -80,7 +80,12 @@ def extract_first_page_text_from_file(filepath: str) -> str: except FileNotFoundError: raise FileNotFoundError(f"PDF file not found: {filepath}") - return extract_first_page_text(pdf_bytes) + return extract_document_text(pdf_bytes) + + +def extract_first_page_text_from_file(filepath: str) -> str: + """Compatibility wrapper for extract_document_text_from_file.""" + return extract_document_text_from_file(filepath) def is_text_empty(text: str, min_length: int = MIN_TEXT_LENGTH) -> bool: @@ -108,7 +113,7 @@ def is_text_empty(text: str, min_length: int = MIN_TEXT_LENGTH) -> bool: def extract_and_validate(pdf_bytes: bytes, min_length: int = MIN_TEXT_LENGTH) -> tuple[str, bool]: - """Extract first page text and check if it's valid. + """Extract document text and check if it's valid. Args: pdf_bytes: The PDF file content as bytes. @@ -121,7 +126,7 @@ def extract_and_validate(pdf_bytes: bytes, min_length: int = MIN_TEXT_LENGTH) -> Raises: TextExtractionError: If extraction fails. """ - text = extract_first_page_text(pdf_bytes) + text = extract_document_text(pdf_bytes) is_valid = not is_text_empty(text, min_length) return (text, is_valid) @@ -142,7 +147,7 @@ class TextExtractor: self.min_length = min_length def extract(self, pdf_bytes: bytes) -> str: - """Extract first page text from PDF bytes. + """Extract text from every page of a PDF. Args: pdf_bytes: The PDF file content as bytes. @@ -150,7 +155,7 @@ class TextExtractor: Returns: The extracted text. """ - return extract_first_page_text(pdf_bytes) + return extract_document_text(pdf_bytes) def extract_and_validate(self, pdf_bytes: bytes) -> tuple[str, bool]: """Extract and validate text. diff --git a/automatic-pdf-renamer/app/pipeline.py b/automatic-pdf-renamer/app/pipeline.py index bd36c66..da2ae0d 100644 --- a/automatic-pdf-renamer/app/pipeline.py +++ b/automatic-pdf-renamer/app/pipeline.py @@ -24,7 +24,7 @@ from .config import ( MappingRule, PeriodFormatConfig, ) -from .extractor import extract_first_page_text, is_text_empty, TextExtractionError +from .extractor import extract_document_text, is_text_empty, TextExtractionError from .period import ( format_period_from_emission_date, ) @@ -99,7 +99,7 @@ class DocumentProcessor: result = ProcessingResult(success=False) try: - # Step 1: Extract first page text + # Step 1: Extract document text if extracted_text is None: extracted_text = self._extract_text(pdf_bytes) @@ -293,7 +293,7 @@ class DocumentProcessor: Extracted text, or None if extraction fails. """ try: - text = extract_first_page_text(pdf_bytes) + text = extract_document_text(pdf_bytes) if is_text_empty(text, self.min_text_length): return None return text diff --git a/automatic-pdf-renamer/scripts/import_paperless.py b/automatic-pdf-renamer/scripts/import_paperless.py index da97e9f..9239236 100755 --- a/automatic-pdf-renamer/scripts/import_paperless.py +++ b/automatic-pdf-renamer/scripts/import_paperless.py @@ -37,7 +37,7 @@ import requests # Add parent directory to path for imports sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from app.extractor import extract_first_page_text +from app.extractor import extract_document_text from app.sidecar import ( Sidecar, SidecarStatus, @@ -826,7 +826,7 @@ class Importer: # Update sidecar sidecar.sha256 = sha256 - sidecar.extracted_text = extract_first_page_text(pdf_bytes) + sidecar.extracted_text = extract_document_text(pdf_bytes) # Write sidecar sidecar_key = get_sidecar_key(target_key) diff --git a/automatic-pdf-renamer/web/app.js b/automatic-pdf-renamer/web/app.js index f734406..758eef8 100644 --- a/automatic-pdf-renamer/web/app.js +++ b/automatic-pdf-renamer/web/app.js @@ -2,8 +2,6 @@ const searchFilter = document.getElementById('search-filter'); const body = document.getElementById('documents-body'); const summaryCards = document.getElementById('summary-cards'); const refreshButton = document.getElementById('refresh-btn'); -const rebuildButton = document.getElementById('rebuild-btn'); -const rebuildStatus = document.getElementById('rebuild-status'); const listStatus = document.getElementById('list-status'); const loadMoreButton = document.getElementById('load-more-btn'); const loadSentinel = document.getElementById('load-sentinel'); @@ -183,32 +181,9 @@ const updateSortIndicators = () => { }); }; -const rebuildDatabase = async () => { - if (!window.confirm('Rebuild the database from sidecar files? Existing database records will be replaced.')) return; - - rebuildButton.disabled = true; - rebuildStatus.textContent = 'Rebuilding database from sidecar files…'; - try { - const response = await fetch('/api/consistency/rebuild', { method: 'POST' }); - if (!response.ok) { - throw new Error(`HTTP ${response.status}`); - } - - const result = await response.json(); - rebuildStatus.textContent = `Database rebuilt from ${result.count} sidecar file${result.count === 1 ? '' : 's'}.`; - await loadDocuments(); - } catch (error) { - rebuildStatus.textContent = 'Unable to rebuild the database. Check the API server and storage backend.'; - console.error(error); - } finally { - rebuildButton.disabled = false; - } -}; - columnFilters.forEach((filter) => filter.addEventListener('input', loadDocuments)); searchFilter.addEventListener('input', loadDocuments); refreshButton.addEventListener('click', loadDocuments); -rebuildButton.addEventListener('click', rebuildDatabase); loadMoreButton.addEventListener('click', () => loadDocuments({ append: true })); sortButtons.forEach((button) => { button.addEventListener('click', () => { diff --git a/automatic-pdf-renamer/web/config.html b/automatic-pdf-renamer/web/config.html index a71e001..3fe34f3 100644 --- a/automatic-pdf-renamer/web/config.html +++ b/automatic-pdf-renamer/web/config.html @@ -15,6 +15,7 @@ diff --git a/automatic-pdf-renamer/web/document.html b/automatic-pdf-renamer/web/document.html index ebd74f7..bc4f090 100644 --- a/automatic-pdf-renamer/web/document.html +++ b/automatic-pdf-renamer/web/document.html @@ -15,6 +15,7 @@ diff --git a/automatic-pdf-renamer/web/document.js b/automatic-pdf-renamer/web/document.js index f738edc..362d45d 100644 --- a/automatic-pdf-renamer/web/document.js +++ b/automatic-pdf-renamer/web/document.js @@ -159,15 +159,25 @@ const buildAdditionalFieldRow = (label = '', value = '', suggestions = {}) => { const row = document.createElement('div'); row.className = 'additional-field-row'; + const labelGroup = document.createElement('div'); + labelGroup.className = 'field-group'; + const labelCaption = document.createElement('label'); + labelCaption.textContent = 'Field name'; const labelSelect = document.createElement('select'); labelSelect.className = 'additional-field-label'; labelSelect.setAttribute('aria-label', 'Additional field name'); + labelGroup.append(labelCaption, labelSelect); const labelValues = [...new Set([...(suggestions.names || []), label].filter((item) => item !== null && item !== undefined && String(item).trim() !== ''))]; setSelectOptions(labelSelect, labelValues, label); + const valueGroup = document.createElement('div'); + valueGroup.className = 'field-group'; + const valueCaption = document.createElement('label'); + valueCaption.textContent = 'Value'; const valueSelect = document.createElement('select'); valueSelect.className = 'additional-field-value'; valueSelect.setAttribute('aria-label', 'Additional field value'); + valueGroup.append(valueCaption, valueSelect); const valueValues = [...new Set([...getFieldValueSuggestions(suggestions, labelSelect.value), value].filter((item) => item !== null && item !== undefined && String(item).trim() !== ''))]; setSelectOptions(valueSelect, valueValues, value); labelSelect.addEventListener('change', () => { @@ -189,7 +199,7 @@ const buildAdditionalFieldRow = (label = '', value = '', suggestions = {}) => { updateSaveButtonState(); }); - row.append(labelSelect, valueSelect, removeButton); + row.append(labelGroup, valueGroup, removeButton); return row; }; diff --git a/automatic-pdf-renamer/web/index.html b/automatic-pdf-renamer/web/index.html index 417f572..3ad0fc3 100644 --- a/automatic-pdf-renamer/web/index.html +++ b/automatic-pdf-renamer/web/index.html @@ -15,6 +15,7 @@ @@ -26,18 +27,15 @@
-
-

-
- - + +
diff --git a/automatic-pdf-renamer/web/maintenance.html b/automatic-pdf-renamer/web/maintenance.html new file mode 100644 index 0000000..6fd084f --- /dev/null +++ b/automatic-pdf-renamer/web/maintenance.html @@ -0,0 +1,60 @@ + + + + + + Maintenance | Automatic PDF Renamer + + + +
+ + +
+
+
+

System tools

+

Maintenance

+
+
+ +
+
+
+

Rebuild database

+

Replace the database index using the metadata sidecar files in storage.

+
+ + +
+ +
+
+

Consistency check

+

Compare database records and metadata sidecars with the files present in storage.

+
+ + +
+
+
+
+ + + + \ No newline at end of file diff --git a/automatic-pdf-renamer/web/maintenance.js b/automatic-pdf-renamer/web/maintenance.js new file mode 100644 index 0000000..c71a555 --- /dev/null +++ b/automatic-pdf-renamer/web/maintenance.js @@ -0,0 +1,49 @@ +const rebuildButton = document.getElementById('rebuild-button'); +const checkButton = document.getElementById('check-button'); +const actionButtons = [rebuildButton, checkButton]; + +const runAction = async ({ operation, progress, status, endpoint, method, confirmation, getResult }) => { + if (!window.confirm(confirmation)) return; + + actionButtons.forEach((actionButton) => { actionButton.disabled = true; }); + operation.hidden = false; + progress.hidden = false; + progress.removeAttribute('value'); + status.textContent = 'Running...'; + + try { + const response = await fetch(endpoint, { method }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const result = await response.json(); + progress.value = 1; + status.textContent = getResult(result); + } catch (error) { + progress.hidden = true; + status.textContent = 'The action failed. Check the API server and storage backend.'; + console.error(error); + } finally { + actionButtons.forEach((actionButton) => { actionButton.disabled = false; }); + } +}; + +rebuildButton.addEventListener('click', () => runAction({ + operation: document.getElementById('rebuild-operation'), + progress: document.getElementById('rebuild-progress'), + status: document.getElementById('rebuild-status'), + endpoint: '/api/consistency/rebuild', + method: 'POST', + confirmation: 'Rebuild the database from sidecar files? Existing database records will be replaced.', + getResult: (result) => `Database rebuilt from ${result.count} sidecar file${result.count === 1 ? '' : 's'}.`, +})); + +checkButton.addEventListener('click', () => runAction({ + operation: document.getElementById('check-operation'), + progress: document.getElementById('check-progress'), + status: document.getElementById('check-status'), + endpoint: '/api/consistency', + method: 'GET', + confirmation: 'Run a consistency check? This scans metadata and database records against storage.', + getResult: (anomalies) => anomalies.length + ? `Consistency check complete: ${anomalies.length} issue${anomalies.length === 1 ? '' : 's'} found.` + : 'Consistency check complete: no issues found.', +})); \ No newline at end of file diff --git a/automatic-pdf-renamer/web/styles.css b/automatic-pdf-renamer/web/styles.css index 6eb48b3..6012808 100644 --- a/automatic-pdf-renamer/web/styles.css +++ b/automatic-pdf-renamer/web/styles.css @@ -172,6 +172,50 @@ body { padding: 18px; } +.maintenance-actions { + display: grid; + gap: 14px; + max-width: 900px; +} + +.maintenance-action { + display: grid; + grid-template-columns: minmax(0, 1fr) max-content; + align-items: center; + gap: 12px 24px; + padding: 18px; + background: var(--panel); + border: 1px solid var(--border); + border-radius: 8px; +} + +.maintenance-action h3 { + margin: 0 0 6px; + font-size: 1.05rem; +} + +.maintenance-action > div:first-child p { + margin: 0; + color: var(--muted); + line-height: 1.5; +} + +.maintenance-operation { + grid-column: 1 / -1; +} + +.maintenance-operation progress { + display: block; + width: 100%; + height: 8px; + accent-color: var(--primary); +} + +.maintenance-operation .action-status { + min-height: 0; + margin: 8px 0 0; +} + .config-description { margin: 0 0 14px; color: var(--muted); @@ -521,13 +565,14 @@ body { .additional-field-row { display: grid; - grid-template-columns: minmax(160px, 1fr) minmax(180px, 1.5fr) auto; - gap: 8px; - align-items: center; + grid-template-columns: minmax(0, 1fr) minmax(0, 1.5fr) auto; + gap: 10px; + align-items: end; } -.additional-field-row input { +.additional-field-row select { width: 100%; + min-width: 0; } .additional-field-empty { @@ -688,6 +733,14 @@ body { grid-template-columns: 1fr; } + .maintenance-action { + grid-template-columns: minmax(0, 1fr); + } + + .maintenance-action > button { + justify-self: start; + } + .main-panel { padding: 18px; }