From b8a8efcfd65725d00ae5d3c70036f808452d1527 Mon Sep 17 00:00:00 2001 From: thomas Date: Wed, 7 Oct 2026 22:14:47 +0200 Subject: [PATCH] First version of the web UI --- automatic-pdf-renamer/.dockerignore | 16 + automatic-pdf-renamer/Dockerfile | 24 + automatic-pdf-renamer/app/api/__init__.py | 5 + automatic-pdf-renamer/app/api/deps.py | 39 + automatic-pdf-renamer/app/api/models.py | 69 ++ automatic-pdf-renamer/app/api/routes.py | 772 ++++++++++++++++++ automatic-pdf-renamer/app/config.py | 51 +- automatic-pdf-renamer/app/db.py | 417 ++++++++++ automatic-pdf-renamer/app/main.py | 78 ++ automatic-pdf-renamer/app/pipeline.py | 13 +- automatic-pdf-renamer/app/sidecar.py | 14 +- automatic-pdf-renamer/main.py | 9 + .../scripts/import_paperless.py | 39 +- ...er-story-0-paperless-ngx-initial-import.md | 0 automatic-pdf-renamer/web/app.js | 236 ++++++ automatic-pdf-renamer/web/document.html | 121 +++ automatic-pdf-renamer/web/document.js | 429 ++++++++++ automatic-pdf-renamer/web/index.html | 86 ++ automatic-pdf-renamer/web/styles.css | 681 +++++++++++++++ 19 files changed, 3052 insertions(+), 47 deletions(-) create mode 100644 automatic-pdf-renamer/.dockerignore create mode 100644 automatic-pdf-renamer/Dockerfile create mode 100644 automatic-pdf-renamer/app/api/__init__.py create mode 100644 automatic-pdf-renamer/app/api/deps.py create mode 100644 automatic-pdf-renamer/app/api/models.py create mode 100644 automatic-pdf-renamer/app/api/routes.py create mode 100644 automatic-pdf-renamer/app/db.py create mode 100644 automatic-pdf-renamer/app/main.py create mode 100644 automatic-pdf-renamer/main.py rename automatic-pdf-renamer/{ => user-stories}/user-story-0-paperless-ngx-initial-import.md (100%) create mode 100644 automatic-pdf-renamer/web/app.js create mode 100644 automatic-pdf-renamer/web/document.html create mode 100644 automatic-pdf-renamer/web/document.js create mode 100644 automatic-pdf-renamer/web/index.html create mode 100644 automatic-pdf-renamer/web/styles.css diff --git a/automatic-pdf-renamer/.dockerignore b/automatic-pdf-renamer/.dockerignore new file mode 100644 index 0000000..89610b0 --- /dev/null +++ b/automatic-pdf-renamer/.dockerignore @@ -0,0 +1,16 @@ +__pycache__/ +*.pyc +*.pyo +*.pyd +.git/ +.gitignore +.venv/ +venv/ +.pytest_cache/ +.mypy_cache/ +.DS_Store +.idea/ +.vscode/ +web/node_modules/ +web/dist/ +*.log diff --git a/automatic-pdf-renamer/Dockerfile b/automatic-pdf-renamer/Dockerfile new file mode 100644 index 0000000..600d06e --- /dev/null +++ b/automatic-pdf-renamer/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.14-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + APP_HOME=/app \ + PORT=8000 + +WORKDIR ${APP_HOME} + +COPY . /app + +RUN python -m pip install --upgrade pip && \ + python -m pip install --no-cache-dir \ + fastapi \ + uvicorn \ + boto3 \ + pymupdf \ + python-dateutil \ + PyYAML + +EXPOSE 8000 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/automatic-pdf-renamer/app/api/__init__.py b/automatic-pdf-renamer/app/api/__init__.py new file mode 100644 index 0000000..de5a60b --- /dev/null +++ b/automatic-pdf-renamer/app/api/__init__.py @@ -0,0 +1,5 @@ +"""API package for the Automatic PDF Renamer service.""" + +from .routes import router + +__all__ = ["router"] diff --git a/automatic-pdf-renamer/app/api/deps.py b/automatic-pdf-renamer/app/api/deps.py new file mode 100644 index 0000000..a261830 --- /dev/null +++ b/automatic-pdf-renamer/app/api/deps.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import os + +from fastapi import Depends, HTTPException, status +from fastapi.security import HTTPBasic, HTTPBasicCredentials + +security = HTTPBasic(auto_error=False) + + +def get_current_user( + credentials: HTTPBasicCredentials | None = Depends(security), +) -> dict[str, str] | None: + """Return the current authenticated user when dashboard auth is configured. + + In v1, auth is optional unless APP_USERNAME and APP_PASSWORD are set. This keeps + local development simple while still enabling a basic password gate in production. + """ + username = os.environ.get("APP_USERNAME") + password = os.environ.get("APP_PASSWORD") + + if not username and not password: + return {"username": "anonymous"} + + if credentials is None: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Authentication required", + headers={"WWW-Authenticate": "Basic realm=automatic-pdf-renamer"}, + ) + + if credentials.username != username or credentials.password != password: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid username or password", + headers={"WWW-Authenticate": "Basic realm=automatic-pdf-renamer"}, + ) + + return {"username": credentials.username} diff --git a/automatic-pdf-renamer/app/api/models.py b/automatic-pdf-renamer/app/api/models.py new file mode 100644 index 0000000..aa61961 --- /dev/null +++ b/automatic-pdf-renamer/app/api/models.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from datetime import date +from typing import Any + +from pydantic import BaseModel, Field + + +class HealthResponse(BaseModel): + """Simple health payload.""" + + status: str = "ok" + app: str = "automatic-pdf-renamer" + + +class FileListItem(BaseModel): + """File metadata returned by the dashboard list endpoint.""" + + id: str + status: str + current_key: str + original_key: str | None = None + template_name: str | None = None + period_format: str | None = None + document_type: str | None = None + emitting_company: str | None = None + period: str | None = None + emission_date: str | None = None + confidence: float | None = None + updated_at: str | None = None + + +class FileDetail(FileListItem): + """Detailed file payload.""" + + schema_version: str | None = None + created_at: str | None = None + extracted_text: str | None = None + optional_fields: dict[str, Any] = Field(default_factory=dict) + history: list[dict[str, Any]] = Field(default_factory=list) + + +class ValidationRequest(BaseModel): + """Body for validating or correcting a file.""" + + template_id: str | None = None + fields: dict[str, Any] = Field(default_factory=dict) + + +class DocumentEditRequest(BaseModel): + """Editable document classification and naming fields.""" + + template_name: str = Field(min_length=1) + period_format: str = Field(min_length=1) + emitting_company: str = Field(min_length=1) + document_type: str = Field(min_length=1) + emission_date: date + optional_fields: dict[str, Any] = Field(default_factory=dict) + + +class ValidationResponse(BaseModel): + """Validation result payload.""" + + id: str + status: str + current_key: str + template_name: str | None = None + fields: dict[str, Any] = Field(default_factory=dict) + updated: bool = False diff --git a/automatic-pdf-renamer/app/api/routes.py b/automatic-pdf-renamer/app/api/routes.py new file mode 100644 index 0000000..f185264 --- /dev/null +++ b/automatic-pdf-renamer/app/api/routes.py @@ -0,0 +1,772 @@ +from __future__ import annotations + +import json +from datetime import datetime, timezone +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException, Request, Response, status + +from app.config import load_templates +from app.db import FileRecord, SQLiteFileRepository +from app.period import format_period_from_emission_date +from app.storage import StorageBackend + +from .deps import get_current_user +from .models import ( + FileDetail, + FileListItem, + DocumentEditRequest, + HealthResponse, + ValidationRequest, + ValidationResponse, +) + +router = APIRouter(tags=["files"]) + + +def _storage(request: Request) -> StorageBackend: + storage = getattr(request.app.state, "storage", None) + if storage is None: + raise HTTPException(status_code=500, detail="Storage backend is not configured") + return storage + + +def _db(request: Request) -> SQLiteFileRepository: + db = getattr(request.app.state, "db", None) + if db is None: + raise HTTPException(status_code=500, detail="SQLite repository is not configured") + return db + + +def _registry(request: Request): + registry = getattr(request.app.state, "template_registry", None) + if registry is None: + registry = load_templates() + request.app.state.template_registry = registry + return registry + + +def _parse_file_metadata(raw: str | bytes | None) -> dict[str, Any]: + if raw is None: + return {} + if isinstance(raw, bytes): + raw = raw.decode("utf-8", errors="replace") + try: + return json.loads(raw) + except json.JSONDecodeError: + return {} + + +def _as_file_item(file_id: str, metadata: dict[str, Any]) -> FileListItem: + return FileListItem( + id=file_id, + status=str(metadata.get("status", "unknown")), + current_key=str(metadata.get("current_key", metadata.get("key", ""))), + original_key=metadata.get("original_key"), + template_name=metadata.get("template_name"), + period_format=metadata.get("period_format"), + document_type=metadata.get("document_type"), + emitting_company=metadata.get("emitting_company"), + period=metadata.get("period"), + emission_date=metadata.get("emission_date"), + confidence=metadata.get("confidence"), + updated_at=metadata.get("updated_at"), + ) + + +@router.get("/health", response_model=HealthResponse) +async def health() -> HealthResponse: + """Service health checkpoint.""" + return HealthResponse(status="ok", app="automatic-pdf-renamer") + + +@router.get("/files", response_model=list[FileListItem]) +async def list_files( + request: Request, + response: Response, + current_user: dict[str, str] | None = Depends(get_current_user), + status: str | None = None, + template: str | None = None, + period_format: str | None = None, + q: str | None = None, + current_key: str | None = None, + emitting_company: str | None = None, + document_type: str | None = None, + period: str | None = None, + confidence: str | None = None, + updated_at: str | None = None, + sort_by: str = "updated_at", + sort_order: str = "desc", + page: int = 1, +) -> list[FileListItem]: + """List documents and report filtered and global counts in response headers.""" + storage = _storage(request) + db = _db(request) + filters = { + "status": status, + "template": template, + "q": q, + "current_key": current_key, + "emitting_company": emitting_company, + "document_type": document_type, + "period": period, + "period_format": period_format, + "confidence": confidence, + "updated_at": updated_at, + } + + if db.count_files() == 0: + for key in sorted(storage.list_objects(prefix="")): + if not key.endswith(".meta.json"): + continue + metadata = _parse_file_metadata(storage.get_object(key)) + if metadata: + db.upsert(FileRecord.from_metadata(metadata)) + + records = db.list_files( + **filters, + sort_by=sort_by, + sort_order=sort_order, + page=page, + page_size=25, + ) + response.headers["X-Total-Count"] = str(db.count_files(**filters)) + response.headers["X-Status-Counts"] = json.dumps(db.get_status_counts()) + + return [ + FileListItem( + id=record.id, + status=record.status, + current_key=record.current_key, + original_key=record.original_key, + template_name=record.template_name, + period_format=record.period_format, + document_type=record.document_type, + emitting_company=record.emitting_company, + period=record.period, + emission_date=record.emission_date, + confidence=record.confidence, + updated_at=record.updated_at, + ) + for record in records + ] + + +@router.get("/files/{file_id}", response_model=FileDetail) +async def get_file( + request: Request, + file_id: str, + current_user: dict[str, str] | None = Depends(get_current_user), +) -> FileDetail: + """Return metadata and extracted text for a single file.""" + storage = _storage(request) + db = _db(request) + record = db.get_by_id(file_id) + + if record is None: + keys = storage.list_objects(prefix="") + for key in keys: + if not key.endswith(".meta.json"): + continue + + metadata = _parse_file_metadata(storage.get_object(key)) + if metadata and (metadata.get("sha256") == file_id or key == file_id): + record = FileRecord.from_metadata(metadata) + db.upsert(record) + break + + if record is None: + raise HTTPException(status_code=404, detail=f"File '{file_id}' not found") + + sidecar_metadata: dict[str, Any] = {} + if record.current_key: + sidecar_metadata = _parse_file_metadata( + storage.get_object(f"{record.current_key}.meta.json") + ) + if not sidecar_metadata or sidecar_metadata.get("sha256") not in { + record.sha256, + record.id, + }: + for key in storage.list_objects(prefix=""): + if not key.endswith(".meta.json"): + continue + candidate = _parse_file_metadata(storage.get_object(key)) + if candidate.get("sha256") in {record.sha256, record.id}: + sidecar_metadata = candidate + break + + metadata = { + "schema_version": sidecar_metadata.get("schema_version"), + "sha256": record.sha256, + "status": record.status, + "current_key": record.current_key, + "original_key": record.original_key, + "template_name": record.template_name, + "period_format": record.period_format, + "document_type": record.document_type, + "emitting_company": record.emitting_company, + "period": record.period, + "emission_date": record.emission_date, + "confidence": record.confidence, + "updated_at": record.updated_at, + "created_at": record.created_at, + "extracted_text": record.extracted_text, + "optional_fields": record.optional_fields, + "history": sidecar_metadata.get("events", []), + } + item = _as_file_item(record.id, metadata) + return FileDetail( + id=item.id, + status=item.status, + current_key=item.current_key, + original_key=item.original_key, + template_name=item.template_name, + period_format=item.period_format, + document_type=item.document_type, + emitting_company=item.emitting_company, + period=item.period, + emission_date=item.emission_date, + confidence=item.confidence, + updated_at=item.updated_at, + schema_version=metadata["schema_version"], + created_at=metadata["created_at"], + extracted_text=record.extracted_text, + optional_fields=record.optional_fields, + history=metadata["history"] if isinstance(metadata["history"], list) else [], + ) + + +@router.get("/files/{file_id}/pdf") +async def get_file_pdf( + request: Request, + file_id: str, + current_user: dict[str, str] | None = Depends(get_current_user), +) -> Response: + """Stream a PDF by key from storage when present.""" + storage = _storage(request) + keys = storage.list_objects(prefix="") + + for key in keys: + if key.endswith(".meta.json"): + metadata = _parse_file_metadata(storage.get_object(key)) + if metadata and (metadata.get("sha256") == file_id or key == file_id): + current_key = metadata.get("current_key") or key.replace(".meta.json", "") + pdf_keys = [current_key] + if not current_key.lower().endswith(".pdf"): + pdf_keys.append(f"{current_key}.pdf") + for pdf_key in pdf_keys: + payload = storage.get_object(pdf_key) + if payload is not None: + return Response(content=payload, media_type="application/pdf") + raise HTTPException(status_code=404, detail="PDF bytes not found for the file") + + raise HTTPException(status_code=404, detail=f"PDF for '{file_id}' not found") + + +@router.post("/files/{file_id}/edit", response_model=FileDetail) +async def edit_file( + request: Request, + file_id: str, + payload: DocumentEditRequest, + current_user: dict[str, str] | None = Depends(get_current_user), +) -> FileDetail: + """Recalculate and persist user-edited document metadata and its storage key.""" + storage = _storage(request) + db = _db(request) + registry = _registry(request) + record = db.get_by_id(file_id) + if record is None: + raise HTTPException(status_code=404, detail=f"File '{file_id}' not found") + + template_name = payload.template_name.strip() + period_format = payload.period_format.strip() + emitting_company = payload.emitting_company.strip() + document_type = payload.document_type.strip() + if not emitting_company or not document_type: + raise HTTPException(status_code=422, detail="Company and document type are required") + + if registry.get_template(template_name) is None: + raise HTTPException(status_code=422, detail=f"Unknown template '{template_name}'") + period_config = registry.get_period_format_config(period_format) + if period_config is None: + raise HTTPException(status_code=422, detail=f"Unknown period format '{period_format}'") + + emission_date = payload.emission_date.isoformat() + try: + period = format_period_from_emission_date( + payload.emission_date, + period_config.to_dict(), + ) + except (TypeError, ValueError) as error: + raise HTTPException(status_code=422, detail=f"Unable to calculate period: {error}") from error + + updated_optional_fields = { + str(label).strip(): value + for label, value in (payload.optional_fields or {}).items() + if str(label).strip() + } + + key_pattern = registry.get_key_pattern(template_name) + if not key_pattern: + raise HTTPException(status_code=422, detail=f"Template '{template_name}' has no key pattern") + pattern_values = dict(updated_optional_fields) + pattern_values.update({ + "company": emitting_company, + "document_type": document_type, + "emission_date": emission_date, + "period": period, + "period_format": period_format, + "template_name": template_name, + "emitting_company": emitting_company, + "original_filename": record.original_key.rsplit("/", 1)[-1], + }) + for _ in range(20): + try: + formatted_key = key_pattern.format(**pattern_values).strip("/") + break + except KeyError as error: + pattern_values[str(error.args[0])] = "unknown" + else: + raise HTTPException(status_code=422, detail="Template key pattern has too many unresolved fields") + + if not formatted_key or any(part in {".", ".."} for part in formatted_key.split("/")): + raise HTTPException(status_code=422, detail="Template produced an invalid document key") + new_current_key = formatted_key if formatted_key.startswith("files/") else f"files/{formatted_key}" + new_pdf_key = new_current_key if new_current_key.lower().endswith(".pdf") else f"{new_current_key}.pdf" + new_sidecar_key = f"{new_current_key}.meta.json" + + source_sidecar_key = "" + sidecar_metadata: dict[str, Any] = {} + known_sidecar_key = f"{record.current_key}.meta.json" if record.current_key else "" + if known_sidecar_key: + candidate = _parse_file_metadata(storage.get_object(known_sidecar_key)) + if candidate and candidate.get("sha256") in {record.sha256, record.id}: + source_sidecar_key = known_sidecar_key + sidecar_metadata = candidate + if not source_sidecar_key: + for key in storage.list_objects(prefix=""): + if not key.endswith(".meta.json"): + continue + candidate = _parse_file_metadata(storage.get_object(key)) + if candidate and ( + candidate.get("sha256") in {record.sha256, record.id} + or candidate.get("current_key") == record.current_key + ): + source_sidecar_key = key + sidecar_metadata = candidate + break + if not source_sidecar_key: + raise HTTPException(status_code=404, detail="Document sidecar was not found") + + source_current_key = str(sidecar_metadata.get("current_key") or record.current_key) + source_pdf_key = "" + pdf_candidates = [source_current_key] + if source_current_key and not source_current_key.lower().endswith(".pdf"): + pdf_candidates.append(f"{source_current_key}.pdf") + for candidate in pdf_candidates: + if candidate and storage.object_exists(candidate): + source_pdf_key = candidate + break + + if storage.object_exists(new_pdf_key) and new_pdf_key != source_pdf_key: + raise HTTPException(status_code=409, detail="The recalculated PDF key is already in use") + destination_sidecar = _parse_file_metadata(storage.get_object(new_sidecar_key)) + if ( + destination_sidecar + and new_sidecar_key != source_sidecar_key + and destination_sidecar.get("sha256") not in {record.sha256, record.id} + ): + raise HTTPException(status_code=409, detail="The recalculated sidecar key is already in use") + + updated_at = datetime.now(timezone.utc).isoformat() + old_values = {} + new_values = {} + if template_name != record.template_name: + old_values["template_name"] = record.template_name + new_values["template_name"] = template_name + if period_format != record.period_format: + old_values["period_format"] = record.period_format + new_values["period_format"] = period_format + if emitting_company != record.emitting_company: + old_values["emitting_company"] = record.emitting_company + new_values["emitting_company"] = emitting_company + if document_type != record.document_type: + old_values["document_type"] = record.document_type + new_values["document_type"] = document_type + if emission_date != record.emission_date: + old_values["emission_date"] = record.emission_date + new_values["emission_date"] = emission_date + if period != record.period: + old_values["period"] = record.period + new_values["period"] = period + if new_pdf_key != record.current_key: + old_values["current_key"] = record.current_key + new_values["current_key"] = new_pdf_key + if updated_optional_fields != record.optional_fields: + old_values["optional_fields"] = dict(record.optional_fields) + new_values["optional_fields"] = dict(updated_optional_fields) + sidecar_metadata.update(new_values) + sidecar_metadata["optional_fields"] = updated_optional_fields + sidecar_metadata["status"] = "validated" + sidecar_metadata["updated_at"] = updated_at + events = sidecar_metadata.get("events") + if not isinstance(events, list): + events = [] + sidecar_metadata["events"] = events + events.append({ + "type": "corrected", + "timestamp": updated_at, + "payload": {"before": old_values, "after": new_values}, + }) + + copied_pdf = bool(source_pdf_key and source_pdf_key != new_pdf_key) + if copied_pdf: + storage.copy_object(source_pdf_key, new_pdf_key) + try: + storage.put_object(new_sidecar_key, json.dumps(sidecar_metadata, ensure_ascii=False, indent=2)) + except Exception: + if copied_pdf: + storage.delete_object(new_pdf_key) + raise + + record.template_name = template_name + record.period_format = period_format + record.emitting_company = emitting_company + record.document_type = document_type + record.emission_date = emission_date + record.period = period + record.current_key = new_pdf_key + record.optional_fields = updated_optional_fields + record.status = "validated" + record.updated_at = updated_at + db.upsert(record) + + if source_sidecar_key != new_sidecar_key: + storage.delete_object(source_sidecar_key) + if source_pdf_key and source_pdf_key != new_pdf_key: + storage.delete_object(source_pdf_key) + + return await get_file(request, record.id, current_user=current_user) + + +@router.post("/files/{file_id}/validate", response_model=ValidationResponse) +async def validate_file( + request: Request, + file_id: str, + payload: ValidationRequest, + current_user: dict[str, str] | None = Depends(get_current_user), +) -> ValidationResponse: + """Apply a manual validation update and persist it back to the sidecar metadata.""" + storage = _storage(request) + db = _db(request) + record = db.get_by_id(file_id) + + if record is None: + keys = storage.list_objects(prefix="") + for key in keys: + if key.endswith(".meta.json"): + metadata = _parse_file_metadata(storage.get_object(key)) + if metadata and (metadata.get("sha256") == file_id or key == file_id): + record = FileRecord.from_metadata(metadata) + db.upsert(record) + break + + if record is None: + raise HTTPException(status_code=404, detail=f"File '{file_id}' not found") + + metadata = dict(record.optional_fields) + if payload.template_id: + record.template_name = payload.template_id + if payload.fields: + record.optional_fields = {**record.optional_fields, **payload.fields} + metadata = record.optional_fields + + record.status = "validated" + record.updated_at = "now" + db.upsert(record) + + sidecar_key = f"{record.current_key}.meta.json" if record.current_key else f"{file_id}.meta.json" + storage.put_object(sidecar_key, json.dumps({ + "sha256": record.sha256, + "original_key": record.original_key, + "current_key": record.current_key, + "status": record.status, + "template_name": record.template_name, + "period_format": record.period_format, + "document_type": record.document_type, + "emitting_company": record.emitting_company, + "period": record.period, + "emission_date": record.emission_date, + "confidence": record.confidence, + "optional_fields": record.optional_fields, + "extracted_text": record.extracted_text, + "updated_at": record.updated_at, + }, ensure_ascii=False, indent=2)) + + return ValidationResponse( + id=str(record.sha256 or file_id), + status="validated", + current_key=record.current_key, + template_name=record.template_name, + fields=record.optional_fields, + updated=True, + ) + + +@router.get("/api/document-options") +async def document_edit_options( + request: Request, + current_user: dict[str, str] | None = Depends(get_current_user), +) -> dict[str, list[str]]: + """Return configured templates and existing document values for edit suggestions.""" + registry = _registry(request) + db = _db(request) + records = db.list_files(page_size=10000) + + companies = sorted({ + str(record.emitting_company).strip() + for record in records + if record.emitting_company and str(record.emitting_company).strip() + }) + document_types = sorted({ + str(record.document_type).strip() + for record in records + if record.document_type and str(record.document_type).strip() + }) + optional_field_names = sorted({ + str(name).strip() + for record in records + for name in (record.optional_fields or {}).keys() + if str(name).strip() + }) + optional_field_values = sorted({ + str(value).strip() + for record in records + for value in (record.optional_fields or {}).values() + if value is not None and str(value).strip() + }) + + return { + "templates": registry.list_all_templates(), + "period_formats": registry.list_period_formats(), + "companies": companies, + "document_types": document_types, + "optional_field_names": optional_field_names, + "optional_field_values": optional_field_values, + } + + +@router.get("/templates") +async def list_templates( + request: Request, + current_user: dict[str, str] | None = Depends(get_current_user), +) -> list[dict[str, Any]]: + """List all templates configured for the service.""" + registry = _registry(request) + items: list[dict[str, Any]] = [] + for template_name in registry.list_all_templates(): + template = registry.get_template(template_name) + if template is None: + continue + items.append({ + "name": template.name, + "key_pattern": template.key_pattern, + }) + return items + + +@router.post("/retrain") +async def start_retrain( + request: Request, + current_user: dict[str, str] | None = Depends(get_current_user), +) -> dict[str, Any]: + """Start a retraining job. The current implementation records a job stub and returns its status.""" + request.app.state.last_retrain_job = { + "job_id": "retrain-1", + "status": "queued", + "message": "Retrain job accepted. Replace with a subprocess-backed job in production.", + } + return request.app.state.last_retrain_job + + +@router.get("/retrain/status") +async def get_retrain_status( + request: Request, + current_user: dict[str, str] | None = Depends(get_current_user), +) -> dict[str, Any]: + """Read the active retraining status.""" + job = getattr(request.app.state, "last_retrain_job", {"job_id": "retrain-1", "status": "idle"}) + return job + + +@router.post("/retrain/{job_id}/activate") +async def activate_model( + job_id: str, + current_user: dict[str, str] | None = Depends(get_current_user), +) -> dict[str, Any]: + """Activate a completed training model.""" + return {"job_id": job_id, "status": "activated", "message": "Model activation accepted."} + + +@router.get("/consistency") +async def list_consistency_flags( + request: Request, + current_user: dict[str, str] | None = Depends(get_current_user), +) -> list[dict[str, Any]]: + """Return consistency anomalies. In the current implementation, this is a lightweight placeholder.""" + storage = _storage(request) + db = _db(request) + anomalies: list[dict[str, Any]] = [] + for key in storage.list_objects(prefix=""): + if key.endswith(".meta.json"): + metadata = _parse_file_metadata(storage.get_object(key)) + if metadata and metadata.get("status") in {"processed", "validated"}: + current_key = metadata.get("current_key", key.replace(".meta.json", "")) + if not storage.object_exists(current_key): + anomalies.append({ + "kind": "missing_object", + "key": key, + "message": "Sidecar points to an object that is no longer present.", + }) + for record in db.list_files(): + if record.status in {"processed", "validated"} and not storage.object_exists(record.current_key): + anomalies.append({ + "kind": "missing_object", + "key": record.current_key, + "message": "SQLite record points to a missing object.", + }) + return anomalies + + +@router.post("/consistency/rebuild") +async def rebuild_consistency( + request: Request, + current_user: dict[str, str] | None = Depends(get_current_user), +) -> dict[str, Any]: + """Rebuild SQLite from sidecar metadata and return the completed result.""" + storage = _storage(request) + db = _db(request) + rebuilt = db.rebuild_from_sidecars(storage) + request.app.state.last_rebuild = { + "status": "completed", + "message": "SQLite rebuilt from sidecar metadata.", + "count": len(rebuilt), + } + return request.app.state.last_rebuild + + +@router.get("/api/health") +async def api_health() -> HealthResponse: + """Alias for /health.""" + return await health() + + +@router.get("/api/files", response_model=list[FileListItem]) +async def list_files_api( + request: Request, + response: Response, + current_user: dict[str, str] | None = Depends(get_current_user), + status: str | None = None, + template: str | None = None, + period_format: str | None = None, + q: str | None = None, + current_key: str | None = None, + emitting_company: str | None = None, + document_type: str | None = None, + period: str | None = None, + confidence: str | None = None, + updated_at: str | None = None, + sort_by: str = "updated_at", + sort_order: str = "desc", + page: int = 1, +) -> list[FileListItem]: + """Compatibility alias for /files.""" + return await list_files( + request=request, + response=response, + current_user=current_user, + status=status, + template=template, + period_format=period_format, + q=q, + current_key=current_key, + emitting_company=emitting_company, + document_type=document_type, + period=period, + confidence=confidence, + updated_at=updated_at, + sort_by=sort_by, + sort_order=sort_order, + page=page, + ) + + +@router.get("/api/files/{file_id}", response_model=FileDetail) +async def get_file_api( + request: Request, + file_id: str, + current_user: dict[str, str] | None = Depends(get_current_user), +) -> FileDetail: + """Compatibility alias for /files/{file_id}.""" + return await get_file(request=request, file_id=file_id, current_user=current_user) + + +@router.post("/api/files/{file_id}/edit", response_model=FileDetail) +async def edit_file_api( + request: Request, + file_id: str, + payload: DocumentEditRequest, + current_user: dict[str, str] | None = Depends(get_current_user), +) -> FileDetail: + """Compatibility alias for POST /files/{file_id}/edit.""" + return await edit_file( + request=request, + file_id=file_id, + payload=payload, + current_user=current_user, + ) + + +@router.get("/api/templates") +async def list_templates_api( + request: Request, + current_user: dict[str, str] | None = Depends(get_current_user), +) -> list[dict[str, Any]]: + """Compatibility alias for /templates.""" + return await list_templates(request=request, current_user=current_user) + + +@router.post("/api/retrain") +async def start_retrain_api( + request: Request, + current_user: dict[str, str] | None = Depends(get_current_user), +) -> dict[str, Any]: + """Compatibility alias for /retrain.""" + return await start_retrain(request=request, current_user=current_user) + + +@router.get("/api/retrain/status") +async def get_retrain_status_api( + request: Request, + current_user: dict[str, str] | None = Depends(get_current_user), +) -> dict[str, Any]: + """Compatibility alias for /retrain/status.""" + return await get_retrain_status(request=request, current_user=current_user) + + +@router.get("/api/consistency") +async def list_consistency_flags_api( + request: Request, + current_user: dict[str, str] | None = Depends(get_current_user), +) -> list[dict[str, Any]]: + """Compatibility alias for /consistency.""" + return await list_consistency_flags(request=request, current_user=current_user) + + +@router.post("/api/consistency/rebuild") +async def rebuild_consistency_api( + request: Request, + current_user: dict[str, str] | None = Depends(get_current_user), +) -> dict[str, Any]: + """Compatibility alias for /consistency/rebuild.""" + return await rebuild_consistency(request=request, current_user=current_user) diff --git a/automatic-pdf-renamer/app/config.py b/automatic-pdf-renamer/app/config.py index cb86c15..5570dca 100644 --- a/automatic-pdf-renamer/app/config.py +++ b/automatic-pdf-renamer/app/config.py @@ -60,31 +60,6 @@ class S3Config: raise ValueError("S3_BUCKET_NAME environment variable is required") -@dataclass -class PaperlessConfig: - """Paperless-ngx API configuration.""" - - url: str - token: str - timeout: int = 30 - - @classmethod - def from_env(cls) -> "PaperlessConfig": - """Create PaperlessConfig from environment variables.""" - return cls( - url=os.environ.get("PAPERLESS_URL", ""), - token=os.environ.get("PAPERLESS_TOKEN", ""), - timeout=int(os.environ.get("PAPERLESS_TIMEOUT", "30")), - ) - - def validate(self) -> None: - """Validate that all required fields are set.""" - if not self.url: - raise ValueError("PAPERLESS_URL environment variable is required") - if not self.token: - raise ValueError("PAPERLESS_TOKEN environment variable is required") - - @dataclass class PollerConfig: """Inbox poller configuration.""" @@ -268,8 +243,9 @@ class TemplateConfig: class AppConfig: """Main application configuration.""" + storage_backend: str = "s3" + local_storage_dir: str = "./storage" s3: S3Config = field(default_factory=S3Config) - paperless: PaperlessConfig | None = None poller: PollerConfig = field(default_factory=PollerConfig) classifier: ClassifierConfig = field(default_factory=ClassifierConfig) debug: bool = False @@ -279,27 +255,32 @@ class AppConfig: def from_env(cls) -> "AppConfig": """Create AppConfig from environment variables.""" debug = os.environ.get("DEBUG", "").lower() in ("true", "1", "yes") + storage_backend = os.environ.get("APP_STORAGE_BACKEND", os.environ.get("STORAGE_BACKEND", "s3")).lower() config = cls( + storage_backend=storage_backend, + local_storage_dir=os.environ.get("LOCAL_STORAGE_DIR", "./storage"), s3=S3Config.from_env(), - paperless=None, # Only needed for import script poller=PollerConfig.from_env(), classifier=ClassifierConfig.from_env(), debug=debug, sqlite_path=os.environ.get("SQLITE_PATH", "./renamer.db"), ) - # Only load paperless config if PAPERLESS_URL is set - if os.environ.get("PAPERLESS_URL"): - config.paperless = PaperlessConfig.from_env() + if config.storage_backend not in {"s3", "file"}: + raise ValueError("STORAGE_BACKEND must be either 's3' or 'file'") return config def validate(self) -> None: """Validate the configuration.""" - self.s3.validate() - if self.paperless: - self.paperless.validate() + if self.storage_backend == "s3": + self.s3.validate() + elif self.storage_backend == "file": + if not self.local_storage_dir: + raise ValueError("LOCAL_STORAGE_DIR is required when STORAGE_BACKEND=file") + else: + raise ValueError("STORAGE_BACKEND must be either 's3' or 'file'") class TemplateRegistry: @@ -617,6 +598,10 @@ class TemplateRegistry: """ return self._period_formats.get(format_name) + def list_period_formats(self) -> list[str]: + """List configured named period formats.""" + return list(self._period_formats.keys()) + def get_key_pattern(self, template_name: str) -> str | None: """Get the key pattern for a template. diff --git a/automatic-pdf-renamer/app/db.py b/automatic-pdf-renamer/app/db.py new file mode 100644 index 0000000..6bac280 --- /dev/null +++ b/automatic-pdf-renamer/app/db.py @@ -0,0 +1,417 @@ +from __future__ import annotations + +import json +import logging +import os +import sqlite3 +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +logger = logging.getLogger("uvicorn.error") + + +@dataclass +class FileRecord: + """SQLite-backed record for a document file.""" + + id: str + sha256: str + original_key: str + current_key: str + status: str = "unknown" + template_name: str | None = None + period_format: str | None = None + document_type: str | None = None + emitting_company: str | None = None + period: str | None = None + emission_date: str | None = None + confidence: float | None = None + optional_fields: dict[str, Any] = field(default_factory=dict) + extracted_text: str | None = None + created_at: str = "" + updated_at: str = "" + + @classmethod + def from_metadata(cls, metadata: dict[str, Any]) -> "FileRecord": + record_id = str(metadata.get("sha256") or metadata.get("id") or metadata.get("current_key") or "unknown") + created_at = metadata.get("created_at") or metadata.get("created") or _utc_now() + updated_at = metadata.get("updated_at") or created_at + return cls( + id=record_id, + sha256=str(metadata.get("sha256") or record_id), + original_key=str(metadata.get("original_key") or metadata.get("source_key") or ""), + current_key=str(metadata.get("current_key") or metadata.get("key") or ""), + status=str(metadata.get("status") or "unknown"), + template_name=metadata.get("template_name"), + period_format=metadata.get("period_format"), + document_type=metadata.get("document_type"), + emitting_company=metadata.get("emitting_company"), + period=metadata.get("period"), + emission_date=metadata.get("emission_date"), + confidence=metadata.get("confidence"), + optional_fields=dict(metadata.get("optional_fields") or {}), + extracted_text=metadata.get("extracted_text"), + created_at=created_at, + updated_at=updated_at, + ) + + def to_row(self) -> tuple[Any, ...]: + return ( + self.id, + self.sha256, + self.original_key, + self.current_key, + self.status, + self.template_name, + self.period_format, + self.document_type, + self.emitting_company, + self.period, + self.emission_date, + self.confidence, + json.dumps(self.optional_fields, ensure_ascii=False), + self.extracted_text, + self.created_at or _utc_now(), + self.updated_at or self.created_at or _utc_now(), + ) + + @classmethod + def from_row(cls, row: tuple[Any, ...]) -> "FileRecord": + ( + id_, + sha256, + original_key, + current_key, + status, + template_name, + period_format, + document_type, + emitting_company, + period, + emission_date, + confidence, + optional_fields, + extracted_text, + created_at, + updated_at, + ) = row + try: + optional_value = json.loads(optional_fields or "{}") if optional_fields else {} + except json.JSONDecodeError: + optional_value = {} + + return cls( + id=str(id_), + sha256=str(sha256), + original_key=str(original_key), + current_key=str(current_key), + status=str(status), + template_name=template_name, + period_format=period_format, + document_type=document_type, + emitting_company=emitting_company, + period=period, + emission_date=emission_date, + confidence=float(confidence) if confidence is not None else None, + optional_fields=dict(optional_value), + extracted_text=extracted_text, + created_at=str(created_at), + updated_at=str(updated_at), + ) + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +class SQLiteFileRepository: + """Repository for storing and querying file metadata in SQLite.""" + + def __init__(self, db_path: str | None = None): + default_path = db_path or os.environ.get("SQLITE_PATH", "./renamer.db") + self.db_path = str(Path(default_path).resolve()) + self._ensure_parent_dir() + self.init_db() + + def _ensure_parent_dir(self) -> None: + directory = Path(self.db_path).parent + directory.mkdir(parents=True, exist_ok=True) + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + return conn + + def init_db(self) -> None: + with self._connect() as conn: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS files ( + id TEXT PRIMARY KEY, + sha256 TEXT NOT NULL UNIQUE, + original_key TEXT, + current_key TEXT, + status TEXT, + template_name TEXT, + period_format TEXT, + document_type TEXT, + emitting_company TEXT, + period TEXT, + emission_date TEXT, + confidence REAL, + optional_fields TEXT, + extracted_text TEXT, + created_at TEXT, + updated_at TEXT + ) + """ + ) + columns = {row[1] for row in conn.execute("PRAGMA table_info(files)")} + if "period_format" not in columns: + conn.execute("ALTER TABLE files ADD COLUMN period_format TEXT") + conn.execute( + """ + CREATE INDEX IF NOT EXISTS idx_files_status + ON files(status) + """ + ) + conn.execute( + """ + CREATE INDEX IF NOT EXISTS idx_files_template + ON files(template_name) + """ + ) + conn.execute( + """ + CREATE INDEX IF NOT EXISTS idx_files_period_format + ON files(period_format) + """ + ) + conn.commit() + + def upsert(self, record: FileRecord) -> FileRecord: + record.updated_at = record.updated_at or _utc_now() + record.created_at = record.created_at or record.updated_at + + with self._connect() as conn: + conn.execute( + """ + INSERT INTO files ( + id, sha256, original_key, current_key, status, template_name, period_format, + document_type, emitting_company, period, emission_date, + confidence, optional_fields, extracted_text, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + sha256 = excluded.sha256, + original_key = excluded.original_key, + current_key = excluded.current_key, + status = excluded.status, + template_name = excluded.template_name, + period_format = excluded.period_format, + document_type = excluded.document_type, + emitting_company = excluded.emitting_company, + period = excluded.period, + emission_date = excluded.emission_date, + confidence = excluded.confidence, + optional_fields = excluded.optional_fields, + extracted_text = excluded.extracted_text, + updated_at = excluded.updated_at + """, + record.to_row(), + ) + conn.commit() + return record + + def get_by_id(self, file_id: str) -> FileRecord | None: + with self._connect() as conn: + row = conn.execute( + "SELECT * FROM files WHERE id = ? OR sha256 = ?", + (file_id, file_id), + ).fetchone() + if row is None: + return None + return FileRecord.from_row(tuple(row)) + + def _file_conditions( + self, + status: str | None = None, + template: str | None = None, + period_format: str | None = None, + q: str | None = None, + current_key: str | None = None, + emitting_company: str | None = None, + document_type: str | None = None, + period: str | None = None, + confidence: str | None = None, + updated_at: str | None = None, + ) -> tuple[list[str], list[Any]]: + conditions: list[str] = [] + params: list[Any] = [] + + if status: + conditions.append("status = ?") + params.append(status) + if template: + conditions.append("LOWER(template_name) LIKE ?") + params.append(f"%{template.lower()}%") + if current_key: + value = f"%{current_key.lower()}%" + conditions.append("(LOWER(current_key) LIKE ? OR LOWER(template_name) LIKE ?)") + params.extend([value, value]) + for column, value in ( + ("emitting_company", emitting_company), + ("document_type", document_type), + ("period", period), + ("updated_at", updated_at), + ("period_format", period_format), + ): + if value: + conditions.append(f"LOWER({column}) LIKE ?") + params.append(f"%{value.lower()}%") + if confidence: + confidence_filter = confidence.strip().removesuffix("%") + if confidence_filter: + conditions.append("CAST(ROUND(confidence * 100, 1) AS TEXT) LIKE ?") + params.append(f"%{confidence_filter.lower()}%") + if q: + conditions.append( + "(LOWER(current_key) LIKE ? OR LOWER(original_key) LIKE ? OR LOWER(template_name) LIKE ? OR LOWER(document_type) LIKE ? OR LOWER(emitting_company) LIKE ? OR LOWER(period) LIKE ? OR LOWER(emission_date) LIKE ? OR LOWER(period_format) LIKE ?)" + ) + like_value = f"%{q.lower()}%" + params.extend([like_value] * 8) + + return conditions, params + + def list_files( + self, + status: str | None = None, + template: str | None = None, + period_format: str | None = None, + q: str | None = None, + current_key: str | None = None, + emitting_company: str | None = None, + document_type: str | None = None, + period: str | None = None, + confidence: str | None = None, + updated_at: str | None = None, + sort_by: str = "updated_at", + sort_order: str = "desc", + page: int = 1, + page_size: int = 25, + ) -> list[FileRecord]: + conditions, params = self._file_conditions( + status=status, + template=template, + period_format=period_format, + q=q, + current_key=current_key, + emitting_company=emitting_company, + document_type=document_type, + period=period, + confidence=confidence, + updated_at=updated_at, + ) + + query = "SELECT * FROM files" + if conditions: + query += " WHERE " + " AND ".join(conditions) + sortable_columns = { + "status": "status", + "current_key": "current_key", + "template_name": "template_name", + "period_format": "period_format", + "emitting_company": "emitting_company", + "document_type": "document_type", + "period": "period", + "confidence": "confidence", + "updated_at": "updated_at", + } + order_column = sortable_columns.get(sort_by, "updated_at") + order_direction = "ASC" if sort_order.lower() == "asc" else "DESC" + query += f" ORDER BY {order_column} {order_direction} LIMIT ? OFFSET ?" + params.extend([page_size, (page - 1) * page_size]) + + with self._connect() as conn: + rows = conn.execute(query, params).fetchall() + + return [FileRecord.from_row(tuple(row)) for row in rows] + + def count_files( + self, + status: str | None = None, + template: str | None = None, + period_format: str | None = None, + q: str | None = None, + current_key: str | None = None, + emitting_company: str | None = None, + document_type: str | None = None, + period: str | None = None, + confidence: str | None = None, + updated_at: str | None = None, + ) -> int: + conditions, params = self._file_conditions( + status=status, + template=template, + period_format=period_format, + q=q, + current_key=current_key, + emitting_company=emitting_company, + document_type=document_type, + period=period, + confidence=confidence, + updated_at=updated_at, + ) + query = "SELECT COUNT(*) FROM files" + if conditions: + query += " WHERE " + " AND ".join(conditions) + with self._connect() as conn: + row = conn.execute(query, params).fetchone() + return int(row[0]) + + def get_status_counts(self) -> dict[str, int]: + with self._connect() as conn: + row = conn.execute( + """ + SELECT + COUNT(*) AS total, + SUM(CASE WHEN status = 'processed' THEN 1 ELSE 0 END) AS processed, + SUM(CASE WHEN status = 'validated' THEN 1 ELSE 0 END) AS validated, + SUM(CASE WHEN status = 'not processed' THEN 1 ELSE 0 END) AS pending + FROM files + """ + ).fetchone() + return {key: int(row[key] or 0) for key in ("total", "processed", "validated", "pending")} + + def delete_all(self) -> None: + with self._connect() as conn: + conn.execute("DELETE FROM files") + conn.commit() + + def rebuild_from_sidecars(self, storage: Any) -> list[FileRecord]: + """Rebuild the SQLite index from JSON sidecar metadata in storage.""" + self.delete_all() + records: list[FileRecord] = [] + + for key in sorted(storage.list_objects(prefix="")): + if not key.endswith(".meta.json"): + continue + raw = storage.get_object(key) + if raw is None: + continue + if isinstance(raw, bytes): + raw = raw.decode("utf-8", errors="replace") + try: + metadata = json.loads(raw) + except json.JSONDecodeError: + continue + if not metadata: + continue + record = FileRecord.from_metadata(metadata) + self.upsert(record) + records.append(record) + + logger.info(f"Rebuilt SQLite index from {len(records)} sidecar metadata files.") + + return records diff --git a/automatic-pdf-renamer/app/main.py b/automatic-pdf-renamer/app/main.py new file mode 100644 index 0000000..88da323 --- /dev/null +++ b/automatic-pdf-renamer/app/main.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import os +from pathlib import Path + +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +from fastapi.staticfiles import StaticFiles + +from app.api.routes import router +from app.config import load_config, load_templates +from app.db import SQLiteFileRepository +from app.storage import get_storage_backend + + +def _build_storage(config) -> object: + backend_name = getattr(config, "storage_backend", "s3").lower() + + if backend_name == "s3": + return get_storage_backend( + "s3", + endpoint_url=config.s3.endpoint_url, + aws_access_key_id=config.s3.access_key, + aws_secret_access_key=config.s3.secret_key, + bucket_name=config.s3.bucket_name, + region_name=config.s3.region, + ) + + storage_dir = getattr(config, "local_storage_dir", os.environ.get("LOCAL_STORAGE_DIR", "./storage")) + Path(storage_dir).mkdir(parents=True, exist_ok=True) + return get_storage_backend("file", base_dir=storage_dir) + + +def create_app() -> FastAPI: + config = load_config() + registry = load_templates(os.environ.get("TEMPLATE_PATH", "./config.jsonc")) + + app = FastAPI( + title="Automatic PDF Renamer", + version="0.1.0", + description="Offline API for PDF classification, validation, retraining, and consistency operations.", + ) + app.state.config = config + app.state.template_registry = registry + app.state.storage = None + app.state.db = SQLiteFileRepository(db_path=config.sqlite_path) + + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + + @app.on_event("startup") + async def startup_event() -> None: + app.state.storage = _build_storage(config) + app.state.template_registry = load_templates( + os.environ.get("TEMPLATE_PATH", "./config.jsonc") + ) + app.state.db = SQLiteFileRepository(db_path=config.sqlite_path) + + app.include_router(router) + + web_dir = Path("./web") + if web_dir.exists(): + app.mount("/", StaticFiles(directory=str(web_dir), html=True), name="web") + app.mount("/static", StaticFiles(directory=str(web_dir)), name="web_static") + + dist_dir = Path("./web/dist") + if dist_dir.exists(): + app.mount("/static", StaticFiles(directory=str(dist_dir)), name="static") + + return app + + +app = create_app() diff --git a/automatic-pdf-renamer/app/pipeline.py b/automatic-pdf-renamer/app/pipeline.py index e132e1a..bd36c66 100644 --- a/automatic-pdf-renamer/app/pipeline.py +++ b/automatic-pdf-renamer/app/pipeline.py @@ -54,6 +54,7 @@ class ProcessingResult: company: str = "" document_type: str = "" template_name: str = "" + period_format: str = "" period: str = "" status: str = SidecarStatus.NOT_PROCESSED.value @@ -159,7 +160,7 @@ class DocumentProcessor: # Extract resolved fields template_name = resolved.get("template") - period_format_name = resolved.get("period_format") + period_format = resolved.get("period_format") additional_field_groups = resolved.get("additional_fields", []) if not template_name: @@ -171,6 +172,7 @@ class DocumentProcessor: return result result.template_name = template_name + result.period_format = period_format # Step 4b: Extract additional fields from additional_fields lists optional_fields = {} @@ -186,7 +188,7 @@ class DocumentProcessor: optional_fields[group_name] = field_value # Step 5: Calculate period using period_format configuration - if not period_format_name: + if not period_format: result.errors.append( f"No period_format resolved for company='{company}', " f"document_type='{doc_type}'. Routing to review." @@ -194,10 +196,10 @@ class DocumentProcessor: result.status = SidecarStatus.NOT_PROCESSED.value return result - period_format_config = self.registry.get_period_format_config(period_format_name) + period_format_config = self.registry.get_period_format_config(period_format) if period_format_config is None: result.errors.append( - f"Period format '{period_format_name}' not found in configuration" + f"Period format '{period_format}' not found in configuration" ) result.status = SidecarStatus.NOT_PROCESSED.value return result @@ -224,6 +226,7 @@ class DocumentProcessor: emission_date=emission_date, period=period, template_name=template_name, + period_format=period_format, emitting_company=company, original_filename=original_key.split("/")[-1], **optional_fields, @@ -244,6 +247,7 @@ class DocumentProcessor: current_key=final_key, status=SidecarStatus.PROCESSED, template_name=template_name, + period_format=period_format, emission_date=emission_date, period=period, emitting_company=company, @@ -263,6 +267,7 @@ class DocumentProcessor: "emission_date": emission_date, "period": period, "confidence": confidence, + "period_format": period_format, "optional_fields": optional_fields, }, ) diff --git a/automatic-pdf-renamer/app/sidecar.py b/automatic-pdf-renamer/app/sidecar.py index 449ba76..3d96305 100644 --- a/automatic-pdf-renamer/app/sidecar.py +++ b/automatic-pdf-renamer/app/sidecar.py @@ -65,6 +65,7 @@ class Sidecar: current_key: str = "" status: str = SidecarStatus.NOT_PROCESSED.value template_name: str = "" + period_format: str = "" confidence: float = 0.0 emission_date: str = "" period: str = "" @@ -105,6 +106,7 @@ class Sidecar: current_key=data.get("current_key", ""), status=data.get("status", SidecarStatus.NOT_PROCESSED.value), template_name=data.get("template_name", ""), + period_format=data.get("period_format", ""), confidence=data.get("confidence", 0.0), emission_date=data.get("emission_date", ""), period=data.get("period", ""), @@ -137,6 +139,7 @@ class Sidecar: # Required common fields per PRD section 4.1 _REQUIRED_COMMON_FIELDS = { "template_name", + "period_format", "emission_date", "emitting_company", "document_type", @@ -150,7 +153,7 @@ _REQUIRED_SIDECAR_FIELDS = { "original_key", "current_key", "status", - *"template_name emission_date emitting_company document_type period".split(), + *"template_name period_format emission_date emitting_company document_type period".split(), "created_at", "updated_at", "events", @@ -236,6 +239,7 @@ def build_sidecar( current_key: str, status: SidecarStatus, template_name: str, + period_format: str, emission_date: str, period: str, emitting_company: str, @@ -252,6 +256,7 @@ def build_sidecar( current_key: Current S3 key of the file. status: Current processing status. template_name: Document family/template name. + period_format: Format of the period string (e.g., "YYYY-MM", "YYYY-MM-DD"). emission_date: Date the document was emitted (ISO format YYYY-MM-DD). period: Human-readable period string. emitting_company: Company that emitted the document. @@ -269,6 +274,7 @@ def build_sidecar( current_key=current_key, status=status.value, template_name=template_name, + period_format=period_format, confidence=confidence, emission_date=emission_date, period=period, @@ -317,8 +323,10 @@ def get_sidecar_key(pdf_key: str) -> str: pdf_key: The S3 key of the PDF file. Returns: - The corresponding sidecar key (e.g., 'files/doc.pdf' -> 'files/doc.pdf.meta.json'). + The corresponding sidecar key (e.g., 'files/doc.pdf' -> 'files/doc.meta.json'). """ + if pdf_key.endswith(".pdf"): + return f"{pdf_key[:-4]}.meta.json" return f"{pdf_key}.meta.json" @@ -345,7 +353,7 @@ def write_atomic( - delete_object(key: str) - object_exists(key: str) -> bool sidecar: The Sidecar to write. - target_key: The final destination key (e.g., 'files/doc.pdf.meta.json'). + target_key: The final destination key (e.g., 'files/doc.meta.json'). temp_prefix: Prefix for temporary keys (default: 'tmp/'). Raises: diff --git a/automatic-pdf-renamer/main.py b/automatic-pdf-renamer/main.py new file mode 100644 index 0000000..7572bbb --- /dev/null +++ b/automatic-pdf-renamer/main.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +import uvicorn + +from app.main import app + + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/automatic-pdf-renamer/scripts/import_paperless.py b/automatic-pdf-renamer/scripts/import_paperless.py index b8b8855..38db0c0 100755 --- a/automatic-pdf-renamer/scripts/import_paperless.py +++ b/automatic-pdf-renamer/scripts/import_paperless.py @@ -62,6 +62,9 @@ logging.basicConfig( ) logger = logging.getLogger(__name__) +PAPERLESS_URL = os.environ.get("PAPERLESS_URL", "") +PAPERLESS_TOKEN = os.environ.get("PAPERLESS_TOKEN", "") + @dataclass class PaperlessDocument: @@ -400,6 +403,29 @@ class DocumentProcessor: self.storage = storage self.pdf_processor = PDFProcessor() + def _unique_target_key(self, target_key: str) -> str: + """Add an incremental suffix when a generated PDF key already exists.""" + pdf_key = f"{target_key}.pdf" + if not self.storage.object_exists(pdf_key) and not self.storage.object_exists( + get_sidecar_key(pdf_key) + ): + return target_key + + directory, separator, filename = target_key.rpartition("/") + extension = ".pdf" if filename.lower().endswith(".pdf") else "" + stem = filename[:-len(extension)] if extension else filename + counter = 2 + + while True: + candidate_filename = f"{stem} ({counter}){extension}" + candidate = f"{directory}{separator}{candidate_filename}" + candidate_pdf_key = f"{candidate}.pdf" + if not self.storage.object_exists(candidate_pdf_key) and not self.storage.object_exists( + get_sidecar_key(candidate_pdf_key) + ): + return candidate + counter += 1 + def resolve_mapping(self, company: str, document_type: str) -> ResolvedMapping: """Resolve the mapping for a document.""" result = ResolvedMapping() @@ -480,7 +506,7 @@ class DocumentProcessor: Returns: Tuple of (target_key, sidecar, errors) - - target_key: Either files/{final_key} or pending/{paperless_id} + - target_key: Either files/{final_key}.pdf or pending/{paperless_id}.pdf - sidecar: Sidecar metadata (or None if failed) - errors: List of error messages """ @@ -593,7 +619,7 @@ class DocumentProcessor: return self._route_to_pending(doc, errors) # Step 7: Build sidecar for files/ - pdf_key = f"files/{final_key}" + pdf_key = self._unique_target_key(f"files/{final_key}.pdf") sha256 = "" # Will be computed when writing sidecar = build_sidecar( @@ -602,6 +628,7 @@ class DocumentProcessor: current_key=pdf_key, status=SidecarStatus.PROCESSED, template_name=template_name, + period_format=resolved.period_format, emission_date=doc.created_date, period=period, emitting_company=company, @@ -629,12 +656,9 @@ class DocumentProcessor: errors: list[str] ) -> tuple[str, Sidecar | None, list[str]]: """Route document to pending/ with error sidecar.""" - pdf_key = f"pending/{doc.id}" + pdf_key = f"pending/{doc.id}.pdf" - # Build sidecar with errors in optional_fields - # We use optional_fields to store errors for pending documents optional_fields = {"tags": doc.tags} if doc.tags else {} - optional_fields["_import_errors"] = errors # Build sidecar sidecar = build_sidecar( @@ -643,6 +667,7 @@ class DocumentProcessor: current_key=pdf_key, status=SidecarStatus.NOT_PROCESSED, template_name="", + period_format="", emission_date=doc.created_date, period="", emitting_company=doc.correspondent_name, @@ -792,7 +817,7 @@ class Importer: return True # Write PDF - self.storage.put_object(target_key + ".pdf", pdf_bytes) + self.storage.put_object(target_key, pdf_bytes) logger.debug(f"Uploaded PDF to {target_key}") # Update sidecar diff --git a/automatic-pdf-renamer/user-story-0-paperless-ngx-initial-import.md b/automatic-pdf-renamer/user-stories/user-story-0-paperless-ngx-initial-import.md similarity index 100% rename from automatic-pdf-renamer/user-story-0-paperless-ngx-initial-import.md rename to automatic-pdf-renamer/user-stories/user-story-0-paperless-ngx-initial-import.md diff --git a/automatic-pdf-renamer/web/app.js b/automatic-pdf-renamer/web/app.js new file mode 100644 index 0000000..f734406 --- /dev/null +++ b/automatic-pdf-renamer/web/app.js @@ -0,0 +1,236 @@ +const searchFilter = document.getElementById('search-filter'); +const body = document.getElementById('documents-body'); +const summaryCards = document.getElementById('summary-cards'); +const refreshButton = document.getElementById('refresh-btn'); +const rebuildButton = document.getElementById('rebuild-btn'); +const rebuildStatus = document.getElementById('rebuild-status'); +const listStatus = document.getElementById('list-status'); +const loadMoreButton = document.getElementById('load-more-btn'); +const loadSentinel = document.getElementById('load-sentinel'); +const columnFilters = [...document.querySelectorAll('[data-filter]')]; +const sortButtons = [...document.querySelectorAll('[data-sort]')]; +let sortBy = 'updated_at'; +let sortOrder = 'desc'; +let page = 1; +let total = 0; +let hasMore = true; +let loading = false; +let reloadAfterRequest = false; +let loadedItems = []; + +const formatUpdated = (value) => { + if (!value) return '—'; + try { + return new Date(value).toLocaleString('fr-FR'); + } catch { + return value; + } +}; + +const escapeHtml = (value) => String(value ?? '').replace(/[&<>"']/g, (character) => ({ + '&': '&', + '<': '<', + '>': '>', + '"': '"', + "'": ''', +}[character])); + +const getListParams = () => { + const params = new URLSearchParams(); + columnFilters.forEach((filter) => { + const value = filter.value.trim(); + if (value) params.set(filter.dataset.filter, value); + }); + const search = searchFilter.value.trim(); + if (search) params.set('q', search); + params.set('sort_by', sortBy); + params.set('sort_order', sortOrder); + return params; +}; + +const badgeClass = (status) => { + const normalized = (status || 'unknown').trim().toLowerCase(); + if (normalized === 'processed' || normalized === 'validated') return normalized; + if (normalized === 'not processed') return 'not processed'; + return 'unknown'; +}; + +const renderSummary = (totals) => { + summaryCards.innerHTML = ` +
+ Total + ${totals.total} +
+
+ Processed + ${totals.processed} +
+
+ Validated + ${totals.validated} +
+
+ Pending + ${totals.pending} +
+ `; +}; + +const renderRows = (items, append) => { + if (!items.length && !append) { + body.innerHTML = 'No documents found.'; + return; + } + + const listParams = getListParams(); + const rows = items + .map((item) => { + const detailParams = new URLSearchParams(listParams); + detailParams.set('id', item.id); + return ` + + ${escapeHtml(item.status || 'unknown')} + + ${escapeHtml(item.current_key?.split('/').pop() || 'Untitled document')} +
${escapeHtml(item.current_key || '—')}
+ + ${escapeHtml(item.template_name || '—')} + ${escapeHtml(item.emitting_company || '—')} + ${escapeHtml(item.document_type || '—')} + ${escapeHtml(item.period || '—')} + ${item.confidence == null ? '—' : `${(Number(item.confidence) * 100).toFixed(1)}%`} + ${escapeHtml(formatUpdated(item.updated_at))} + + `; + }) + .join(''); + if (append) { + body.insertAdjacentHTML('beforeend', rows); + } else { + body.innerHTML = rows; + } +}; + +const loadDocuments = async ({ append = false } = {}) => { + if (loading) { + if (!append) reloadAfterRequest = true; + return; + } + if (append && !hasMore) return; + loading = true; + loadMoreButton.disabled = true; + if (!append) { + page = 1; + total = 0; + hasMore = true; + loadedItems = []; + listStatus.textContent = 'Loading documents...'; + } else { + listStatus.textContent = `Loading more documents (${loadedItems.length} of ${total})...`; + } + + const params = getListParams(); + params.set('page', String(page)); + + try { + const response = await fetch(`/api/files?${params.toString()}`); + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + + const items = await response.json(); + total = Number(response.headers.get('X-Total-Count')) || 0; + const counts = JSON.parse(response.headers.get('X-Status-Counts') || '{}'); + renderSummary({ + total: counts.total ?? total, + processed: counts.processed ?? 0, + validated: counts.validated ?? 0, + pending: counts.pending ?? 0, + }); + renderRows(items, append); + loadedItems = append ? loadedItems.concat(items) : items; + page += 1; + hasMore = loadedItems.length < total; + listStatus.textContent = total + ? `Showing ${loadedItems.length} of ${total} matching documents.` + : 'No documents match these filters.'; + loadMoreButton.hidden = !hasMore; + } catch (error) { + if (!append) { + body.innerHTML = 'Unable to load documents. Check the API server.'; + summaryCards.innerHTML = '
Error!
'; + } + listStatus.textContent = append + ? `Unable to load more documents. ${loadedItems.length} of ${total} loaded.` + : 'Unable to load documents. Check the API server.'; + console.error(error); + } finally { + loading = false; + loadMoreButton.disabled = false; + if (reloadAfterRequest) { + reloadAfterRequest = false; + loadDocuments(); + } + } +}; + +const updateSortIndicators = () => { + document.querySelectorAll('[data-sort-column]').forEach((header) => { + const active = header.dataset.sortColumn === sortBy; + header.setAttribute('aria-sort', active ? (sortOrder === 'asc' ? 'ascending' : 'descending') : 'none'); + const indicator = header.querySelector('.sort-button span'); + indicator.textContent = active ? (sortOrder === 'asc' ? '↑' : '↓') : ''; + }); +}; + +const rebuildDatabase = async () => { + if (!window.confirm('Rebuild the database from sidecar files? Existing database records will be replaced.')) return; + + rebuildButton.disabled = true; + rebuildStatus.textContent = 'Rebuilding database from sidecar files…'; + try { + const response = await fetch('/api/consistency/rebuild', { method: 'POST' }); + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + + const result = await response.json(); + rebuildStatus.textContent = `Database rebuilt from ${result.count} sidecar file${result.count === 1 ? '' : 's'}.`; + await loadDocuments(); + } catch (error) { + rebuildStatus.textContent = 'Unable to rebuild the database. Check the API server and storage backend.'; + console.error(error); + } finally { + rebuildButton.disabled = false; + } +}; + +columnFilters.forEach((filter) => filter.addEventListener('input', loadDocuments)); +searchFilter.addEventListener('input', loadDocuments); +refreshButton.addEventListener('click', loadDocuments); +rebuildButton.addEventListener('click', rebuildDatabase); +loadMoreButton.addEventListener('click', () => loadDocuments({ append: true })); +sortButtons.forEach((button) => { + button.addEventListener('click', () => { + if (sortBy === button.dataset.sort) { + sortOrder = sortOrder === 'asc' ? 'desc' : 'asc'; + } else { + sortBy = button.dataset.sort; + sortOrder = 'asc'; + } + updateSortIndicators(); + loadDocuments(); + }); +}); + +updateSortIndicators(); +loadDocuments(); + +if ('IntersectionObserver' in window) { + const loadObserver = new IntersectionObserver((entries) => { + if (entries.some((entry) => entry.isIntersecting)) { + loadDocuments({ append: true }); + } + }, { rootMargin: '300px' }); + loadObserver.observe(loadSentinel); +} diff --git a/automatic-pdf-renamer/web/document.html b/automatic-pdf-renamer/web/document.html new file mode 100644 index 0000000..f8ac070 --- /dev/null +++ b/automatic-pdf-renamer/web/document.html @@ -0,0 +1,121 @@ + + + + + + Document | Automatic PDF Renamer + + + +
+ + +
+
+
+

Document detail

+

Loading document...

+
+
+ + Back to documents +
+
+ +

+ + +
+
+ + + + \ No newline at end of file diff --git a/automatic-pdf-renamer/web/document.js b/automatic-pdf-renamer/web/document.js new file mode 100644 index 0000000..e32d734 --- /dev/null +++ b/automatic-pdf-renamer/web/document.js @@ -0,0 +1,429 @@ +const title = document.getElementById('document-title'); +const statusMessage = document.getElementById('detail-status'); +const content = document.getElementById('document-content'); +const technicalDataList = document.getElementById('technical-data'); +const additionalSection = document.getElementById('additional-section'); +const additionalFields = document.getElementById('additional-fields'); +const addAdditionalFieldButton = document.getElementById('add-additional-field'); +const historyList = document.getElementById('document-history'); +const extractedText = document.getElementById('extracted-text'); +const pdfPreview = document.getElementById('pdf-preview'); +const openPdf = document.getElementById('open-pdf'); +const previousButton = document.getElementById('previous-document'); +const nextButton = document.getElementById('next-document'); +const documentPosition = document.getElementById('document-position'); +const editForm = document.getElementById('document-edit-form'); +const templateSelect = document.getElementById('edit-template'); +const periodFormatSelect = document.getElementById('edit-period-format'); +const companyInput = document.getElementById('edit-company'); +const documentTypeInput = document.getElementById('edit-document-type'); +const emissionDateInput = document.getElementById('edit-emission-date'); +const periodPreview = document.getElementById('period-preview'); +const keyPreview = document.getElementById('current-key-preview'); +const saveButton = document.getElementById('save-document-button'); +const editStatus = document.getElementById('edit-status'); +saveButton.disabled = true; +let currentFileId = null; +let fieldSuggestionState = { names: [], values: [] }; +let initialDocumentState = null; + +const canonicalizeState = (value) => { + if (value === null || value === undefined) return ''; + if (typeof value === 'string') return value.trim(); + if (Array.isArray(value)) return value.map((item) => canonicalizeState(item)); + if (typeof value === 'object') { + return Object.fromEntries( + Object.entries(value) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, nestedValue]) => [key, canonicalizeState(nestedValue)]), + ); + } + return value; +}; + +const getCurrentDocumentState = () => ({ + template_name: templateSelect.value || '', + period_format: periodFormatSelect.value || '', + emitting_company: companyInput.value.trim(), + document_type: documentTypeInput.value.trim(), + emission_date: parseEmissionDateInput(emissionDateInput.value) || '', + optional_fields: canonicalizeState(getAdditionalFieldEntries()), +}); + +const updateSaveButtonState = () => { + if (!currentFileId || !initialDocumentState) { + saveButton.disabled = true; + return; + } + const currentState = canonicalizeState(getCurrentDocumentState()); + const baselineState = canonicalizeState(initialDocumentState); + saveButton.disabled = JSON.stringify(currentState) === JSON.stringify(baselineState); +}; + +const openNeighbor = (neighborId) => { + if (!neighborId) return; + const params = new URLSearchParams(window.location.search); + params.set('id', neighborId); + window.location.href = `/document.html?${params.toString()}`; +}; + +const loadDocumentNavigation = async (fileId) => { + previousButton.disabled = true; + nextButton.disabled = true; + const filters = new URLSearchParams(window.location.search); + filters.delete('id'); + filters.delete('page'); + if (!filters.has('sort_by')) filters.set('sort_by', 'updated_at'); + if (!filters.has('sort_order')) filters.set('sort_order', 'desc'); + + let previous = null; + let page = 1; + let total = 0; + try { + while (true) { + const params = new URLSearchParams(filters); + params.set('page', String(page)); + const response = await fetch(`/api/files?${params.toString()}`); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const items = await response.json(); + total = Number(response.headers.get('X-Total-Count')) || total; + const index = items.findIndex((item) => String(item.id) === fileId); + + if (index !== -1) { + const preceding = index > 0 ? items[index - 1] : previous; + let following = index + 1 < items.length ? items[index + 1] : null; + const position = (page - 1) * 25 + index + 1; + if (!following && position < total) { + const nextParams = new URLSearchParams(filters); + nextParams.set('page', String(page + 1)); + const nextResponse = await fetch(`/api/files?${nextParams.toString()}`); + if (!nextResponse.ok) throw new Error(`HTTP ${nextResponse.status}`); + following = (await nextResponse.json())[0] || null; + } + + previousButton.disabled = !preceding; + nextButton.disabled = !following; + previousButton.onclick = () => openNeighbor(preceding?.id); + nextButton.onclick = () => openNeighbor(following?.id); + documentPosition.textContent = `${position} of ${total}`; + return; + } + + if (items.length < 25 || page * 25 >= total) break; + previous = items[items.length - 1] || previous; + page += 1; + } + documentPosition.textContent = 'Not in this list'; + } catch (error) { + documentPosition.textContent = 'Navigation unavailable'; + console.error(error); + } +}; + +const formatDate = (value) => { + if (!value) return 'Not recorded'; + const date = new Date(value); + return Number.isNaN(date.getTime()) ? value : date.toLocaleString('fr-FR'); +}; + +const formatEmissionDateInput = (value) => { + const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value || ''); + return match ? `${match[3]}/${match[2]}/${match[1]}` : ''; +}; + +const parseEmissionDateInput = (value) => { + const match = /^(\d{2})\/(\d{2})\/(\d{4})$/.exec(value.trim()); + if (!match) return null; + const [, day, month, year] = match; + const isoDate = `${year}-${month}-${day}`; + const parsedDate = new Date(`${isoDate}T00:00:00Z`); + if (Number.isNaN(parsedDate.getTime()) || parsedDate.toISOString().slice(0, 10) !== isoDate) return null; + return isoDate; +}; + +const addMetadata = (target, label, value) => { + if (value === null || value === undefined || value === '') return; + const term = document.createElement('dt'); + term.textContent = label; + const description = document.createElement('dd'); + description.textContent = typeof value === 'object' ? JSON.stringify(value, null, 2) : String(value); + target.append(term, description); +}; + +const buildAdditionalFieldRow = (label = '', value = '', suggestions = {}) => { + const row = document.createElement('div'); + row.className = 'additional-field-row'; + + const labelSelect = document.createElement('select'); + labelSelect.className = 'additional-field-label'; + labelSelect.setAttribute('aria-label', 'Additional field name'); + const labelValues = [...new Set([...(suggestions.names || []), label].filter((item) => item !== null && item !== undefined && String(item).trim() !== ''))]; + setSelectOptions(labelSelect, labelValues, label); + + const valueSelect = document.createElement('select'); + valueSelect.className = 'additional-field-value'; + valueSelect.setAttribute('aria-label', 'Additional field value'); + const valueValues = [...new Set([...(suggestions.values || []), value].filter((item) => item !== null && item !== undefined && String(item).trim() !== ''))]; + setSelectOptions(valueSelect, valueValues, value); + + const removeButton = document.createElement('button'); + removeButton.type = 'button'; + removeButton.className = 'secondary-btn small-btn'; + removeButton.textContent = 'Remove'; + removeButton.addEventListener('click', () => { + row.remove(); + if (!additionalFields.querySelector('.additional-field-row')) { + const empty = document.createElement('p'); + empty.className = 'additional-field-empty'; + empty.textContent = 'No custom fields yet.'; + additionalFields.append(empty); + } + updateSaveButtonState(); + }); + + row.append(labelSelect, valueSelect, removeButton); + return row; +}; + +const renderAdditionalFields = (fields, suggestions = {}) => { + additionalFields.replaceChildren(); + const entries = Object.entries(fields || {}); + if (!entries.length) { + const empty = document.createElement('p'); + empty.className = 'additional-field-empty'; + empty.textContent = 'No custom fields yet.'; + additionalFields.append(empty); + return; + } + + entries.forEach(([label, value]) => { + additionalFields.append(buildAdditionalFieldRow(label, value, suggestions)); + }); +}; + +const updateDocumentSuggestions = (options = {}) => { + setSelectOptions(companyInput, options.companies || [], companyInput.value || ''); + setSelectOptions(documentTypeInput, options.document_types || [], documentTypeInput.value || ''); + + const suggestionState = { + names: options.optional_field_names || [], + values: options.optional_field_values || [], + }; + + additionalFields.querySelectorAll('.additional-field-row').forEach((row) => { + const labelSelect = row.querySelector('.additional-field-label'); + const valueSelect = row.querySelector('.additional-field-value'); + if (labelSelect) { + const current = labelSelect.value || ''; + setSelectOptions(labelSelect, suggestionState.names, current); + } + if (valueSelect) { + const current = valueSelect.value || ''; + setSelectOptions(valueSelect, suggestionState.values, current); + } + }); + + return suggestionState; +}; + +const getAdditionalFieldEntries = () => { + const fields = {}; + additionalFields.querySelectorAll('.additional-field-row').forEach((row) => { + const labelSelect = row.querySelector('.additional-field-label'); + const valueSelect = row.querySelector('.additional-field-value'); + const label = (labelSelect?.value || '').trim(); + if (!label) return; + fields[label] = valueSelect?.value ?? ''; + }); + return fields; +}; + +const setSelectOptions = (select, values, selectedValue) => { + select.replaceChildren(); + const options = [...values]; + if (selectedValue && !options.includes(selectedValue)) options.unshift(selectedValue); + if (!options.length) { + const placeholder = document.createElement('option'); + placeholder.value = ''; + placeholder.textContent = 'Select...'; + select.append(placeholder); + select.value = ''; + return; + } + options.forEach((value) => { + const option = document.createElement('option'); + option.value = value; + option.textContent = value; + select.append(option); + }); + select.value = selectedValue || options[0] || ''; +}; + +const renderHistory = (events) => { + historyList.replaceChildren(); + if (!events.length) { + const empty = document.createElement('li'); + empty.className = 'history-empty'; + empty.textContent = 'No history has been recorded for this document.'; + historyList.append(empty); + return; + } + + [...events].reverse().forEach((event) => { + const entry = document.createElement('li'); + entry.className = 'history-entry'; + const heading = document.createElement('div'); + heading.className = 'history-heading'; + const kind = document.createElement('strong'); + kind.textContent = String(event.type || 'Update').replaceAll('_', ' '); + const timestamp = document.createElement('time'); + timestamp.textContent = formatDate(event.timestamp); + heading.append(kind, timestamp); + entry.append(heading); + + if (event.payload && Object.keys(event.payload).length) { + const payload = document.createElement('pre'); + payload.textContent = JSON.stringify(event.payload, null, 2); + entry.append(payload); + } + historyList.append(entry); + }); +}; + +const renderDocument = (documentData, fileId) => { + currentFileId = fileId; + title.textContent = documentData.current_key.split('/').pop() || 'Document'; + technicalDataList.replaceChildren(); + additionalFields.replaceChildren(); + templateSelect.value = documentData.template_name || ''; + periodFormatSelect.value = documentData.period_format || ''; + companyInput.value = documentData.emitting_company || ''; + documentTypeInput.value = documentData.document_type || ''; + emissionDateInput.value = formatEmissionDateInput(documentData.emission_date); + periodPreview.textContent = documentData.period || '—'; + keyPreview.textContent = documentData.current_key || '—'; + + renderAdditionalFields(documentData.optional_fields || {}, fieldSuggestionState); + + initialDocumentState = { + template_name: templateSelect.value || '', + period_format: periodFormatSelect.value || '', + emitting_company: (companyInput.value || '').trim(), + document_type: (documentTypeInput.value || '').trim(), + emission_date: parseEmissionDateInput(emissionDateInput.value) || '', + optional_fields: canonicalizeState(getAdditionalFieldEntries()), + }; + updateSaveButtonState(); + + const technicalMetadata = [ + ['Status', documentData.status], + ['Confidence', documentData.confidence], + ['Original key', documentData.original_key], + ['Current key', documentData.current_key], + ['SHA-256', documentData.id], + ['Created', formatDate(documentData.created_at)], + ['Updated', formatDate(documentData.updated_at)], + ['Schema version', documentData.schema_version], + ]; + technicalMetadata.forEach(([label, value]) => addMetadata(technicalDataList, label, value)); + + renderHistory(Array.isArray(documentData.history) ? documentData.history : []); + extractedText.textContent = documentData.extracted_text || 'No extracted text is available.'; + const pdfUrl = `/files/${encodeURIComponent(fileId)}/pdf`; + pdfPreview.src = pdfUrl; + openPdf.href = pdfUrl; + content.hidden = false; + statusMessage.textContent = ''; +}; + +const saveDocument = async (event) => { + event.preventDefault(); + if (!currentFileId || saveButton.disabled) return; + const emissionDate = parseEmissionDateInput(emissionDateInput.value); + emissionDateInput.setCustomValidity(emissionDate ? '' : 'Enter a valid date in dd/mm/yyyy format.'); + if (!editForm.reportValidity()) return; + + saveButton.disabled = true; + editStatus.textContent = 'Recalculating document name and saving...'; + const payload = { + template_name: templateSelect.value, + period_format: periodFormatSelect.value, + emitting_company: companyInput.value.trim(), + document_type: documentTypeInput.value.trim(), + emission_date: emissionDate, + optional_fields: getAdditionalFieldEntries(), + }; + try { + const response = await fetch(`/api/files/${encodeURIComponent(currentFileId)}/edit`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }); + const result = await response.json(); + if (!response.ok) throw new Error(result.detail || `HTTP ${response.status}`); + renderDocument(result, currentFileId); + editStatus.textContent = 'Changes saved. The document is validated.'; + updateSaveButtonState(); + await loadDocumentNavigation(currentFileId); + } catch (error) { + editStatus.textContent = `Unable to save changes: ${error.message}`; + console.error(error); + } finally { + saveButton.disabled = false; + } +}; + +if (addAdditionalFieldButton) { + addAdditionalFieldButton.addEventListener('click', () => { + const emptyState = additionalFields.querySelector('.additional-field-empty'); + if (emptyState) emptyState.remove(); + additionalFields.append(buildAdditionalFieldRow('', '', fieldSuggestionState)); + updateSaveButtonState(); + }); +} + +editForm.addEventListener('input', updateSaveButtonState); +editForm.addEventListener('change', updateSaveButtonState); +additionalFields.addEventListener('input', updateSaveButtonState); +additionalFields.addEventListener('change', updateSaveButtonState); + +const loadDocument = async () => { + const fileId = new URLSearchParams(window.location.search).get('id'); + if (!fileId) { + title.textContent = 'Document not specified'; + statusMessage.textContent = 'Return to the document list and select a document.'; + return; + } + + try { + const [documentResponse, optionsResponse] = await Promise.all([ + fetch(`/api/files/${encodeURIComponent(fileId)}`), + fetch('/api/document-options'), + ]); + if (!documentResponse.ok) throw new Error(`HTTP ${documentResponse.status}`); + if (!optionsResponse.ok) throw new Error(`Unable to load edit options (HTTP ${optionsResponse.status})`); + const [documentData, options] = await Promise.all([ + documentResponse.json(), + optionsResponse.json(), + ]); + fieldSuggestionState = { + names: options.optional_field_names || [], + values: options.optional_field_values || [], + }; + setSelectOptions(templateSelect, options.templates || [], documentData.template_name); + setSelectOptions(periodFormatSelect, options.period_formats || [], documentData.period_format); + setSelectOptions(companyInput, options.companies || [], documentData.emitting_company); + setSelectOptions(documentTypeInput, options.document_types || [], documentData.document_type); + renderDocument(documentData, fileId); + renderAdditionalFields(documentData.optional_fields || {}, fieldSuggestionState); + updateSaveButtonState(); + await loadDocumentNavigation(fileId); + } catch (error) { + title.textContent = 'Unable to load document'; + statusMessage.textContent = 'The document could not be loaded. It may have been removed or the API may be unavailable.'; + console.error(error); + } +}; + +editForm.addEventListener('submit', saveDocument); +loadDocument(); \ No newline at end of file diff --git a/automatic-pdf-renamer/web/index.html b/automatic-pdf-renamer/web/index.html new file mode 100644 index 0000000..0b50f12 --- /dev/null +++ b/automatic-pdf-renamer/web/index.html @@ -0,0 +1,86 @@ + + + + + + Automatic PDF Renamer + + + +
+ + +
+
+
+

Dashboard

+

Document list

+
+
+ + +
+
+ +

+ +
+ +
+
+ + +
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Loading documents...
+
+
+

Loading documents...

+ + +
+
+
+ + + + diff --git a/automatic-pdf-renamer/web/styles.css b/automatic-pdf-renamer/web/styles.css new file mode 100644 index 0000000..d41e4de --- /dev/null +++ b/automatic-pdf-renamer/web/styles.css @@ -0,0 +1,681 @@ +:root { + --bg: #f2f5f9; + --panel: #ffffff; + --sidebar: #12263a; + --sidebar-text: #d7e5f6; + --muted: #61758a; + --primary: #2463eb; + --primary-soft: #eaf1ff; + --border: #dfe8f3; + --success: #1d8f5f; + --warning: #b77810; + --danger: #c53d3d; + --text: #1d2a36; +} + +* { + box-sizing: border-box; +} + +html, body { + margin: 0; + font-family: Inter, "Segoe UI", sans-serif; + background: var(--bg); + color: var(--text); +} + +body { + min-height: 100vh; +} + +.app-shell { + display: flex; + min-height: 100vh; +} + +.sidebar { + width: 240px; + background: var(--sidebar); + color: var(--sidebar-text); + padding: 28px 18px; +} + +.brand h1 { + margin: 0; + font-size: 1.4rem; + letter-spacing: 0.02em; +} + +.nav { + margin-top: 28px; + display: flex; + flex-direction: column; + gap: 8px; +} + +.nav a { + color: var(--sidebar-text); + text-decoration: none; + background: rgba(255,255,255,0.04); + border-radius: 10px; + padding: 11px 12px; + font-weight: 600; +} + +.nav a.active { + background: rgba(255,255,255,0.12); +} + +.main-panel { + flex: 1; + padding: 30px; +} + +.toolbar { + display: flex; + align-items: center; + justify-content: space-between; + margin-bottom: 22px; +} + +.eyebrow { + text-transform: uppercase; + letter-spacing: 0.08em; + font-size: 0.72rem; + color: var(--muted); + margin: 0 0 4px; +} + +.toolbar h2 { + margin: 0; + font-size: 2rem; +} + +.primary-btn { + background: var(--primary); + border: none; + color: white; + border-radius: 10px; + padding: 10px 16px; + cursor: pointer; + font-weight: 600; +} + +.toolbar-actions { + display: flex; + align-items: center; + gap: 10px; +} + +.secondary-btn { + background: var(--panel); + border: 1px solid var(--border); + color: var(--text); + border-radius: 8px; + padding: 9px 14px; + cursor: pointer; + font-weight: 600; +} + +.primary-btn:disabled, +.secondary-btn:disabled { + cursor: wait; + opacity: 0.65; +} + +.action-status { + min-height: 1.25rem; + margin: -12px 0 14px; + color: var(--muted); + font-size: 0.9rem; +} + +.summary-cards { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); + gap: 16px; + margin-bottom: 22px; +} + +.summary-card { + background: var(--panel); + border: 1px solid var(--border); + border-radius: 12px; + padding: 18px; + box-shadow: 0 3px 10px rgba(26, 44, 64, 0.03); +} + +.summary-card .label { + color: var(--muted); + font-size: 0.78rem; + text-transform: uppercase; + letter-spacing: 0.06em; +} + +.summary-card .value { + display: block; + margin-top: 12px; + font-size: 1.9rem; + font-weight: 700; +} + +.filters-card, +.table-card { + background: var(--panel); + border: 1px solid var(--border); + border-radius: 12px; + box-shadow: 0 3px 10px rgba(26, 44, 64, 0.03); +} + +.filters-card { + display: grid; + grid-template-columns: minmax(220px, 1fr); + gap: 18px; + padding: 18px; + margin-bottom: 22px; +} + +.field-group { + display: flex; + flex-direction: column; + gap: 8px; +} + +.field-group label { + font-weight: 600; + color: var(--muted); +} + +.field-group input, +.field-group select { + border: 1px solid var(--border); + border-radius: 8px; + height: 40px; + padding: 0 12px; + font-size: 0.96rem; +} + +.table-card { + overflow-x: auto; +} + +.table-card table { + width: 100%; + border-collapse: collapse; +} + +.documents-table { + min-width: 1120px; +} + +.table-card th, +.table-card td { + padding: 14px 16px; + border-bottom: 1px solid var(--border); + text-align: left; + vertical-align: top; +} + +.table-card th { + background: #f8fafc; + font-size: 0.78rem; + letter-spacing: 0.04em; + color: var(--muted); + text-transform: uppercase; +} + +.sort-button { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 0; + border: 0; + background: transparent; + color: inherit; + font: inherit; + text-transform: inherit; + cursor: pointer; +} + +.sort-button:hover { + color: var(--primary); +} + +.sort-button span { + min-width: 0.75em; + color: var(--primary); +} + +.column-filters th { + padding: 8px; + background: var(--panel); +} + +.column-filters input, +.column-filters select { + width: 100%; + min-width: 92px; + height: 34px; + padding: 0 8px; + border: 1px solid var(--border); + border-radius: 5px; + background: white; + color: var(--text); + font: inherit; + font-size: 0.82rem; +} + +.visually-hidden { + position: absolute; + width: 1px; + height: 1px; + padding: 0; + margin: -1px; + overflow: hidden; + clip: rect(0, 0, 0, 0); + white-space: nowrap; + border: 0; +} + +.badge { + display: inline-flex; + align-items: center; + border-radius: 999px; + padding: 5px 10px; + font-size: 0.74rem; + font-weight: 700; + background: #edf2ff; + color: var(--primary); + text-transform: capitalize; +} + +.badge.processed { + background: #e7f8ee; + color: var(--success); +} + +.badge.validated { + background: #e7f8ee; + color: var(--success); +} + +.badge.not\ processed { + background: #fff2da; + color: var(--warning); +} + +.badge.unknown { + background: #f4f5f7; + color: var(--muted); +} + +.empty-state { + text-align: center; + color: var(--muted); + padding: 28px 12px; +} + +.list-footer { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + flex-wrap: wrap; + padding: 12px 2px 28px; +} + +.list-footer .action-status { + flex: 1 1 220px; + min-height: 0; + margin: 0; +} + +#load-sentinel { + height: 1px; +} + +.key-text { + font-family: "SFMono-Regular", Consolas, monospace; + font-size: 0.8rem; + color: var(--muted); +} + +.document-link { + color: var(--primary); + font-weight: 650; + text-decoration: none; +} + +.document-link:hover, +.section-heading a:hover { + text-decoration: underline; +} + +.detail-toolbar h2 { + max-width: min(70vw, 900px); + overflow-wrap: anywhere; + font-size: 1.55rem; +} + +.detail-toolbar-actions, +.document-navigation { + display: flex; + align-items: center; + gap: 10px; +} + +.detail-toolbar-actions { + flex-wrap: wrap; + justify-content: flex-end; +} + +.document-navigation span { + min-width: 72px; + color: var(--muted); + font-size: 0.82rem; + text-align: center; + white-space: nowrap; +} + +.document-navigation button:disabled { + cursor: default; + opacity: 0.5; +} + +.back-link { + display: inline-flex; + align-items: center; + min-height: 40px; + text-decoration: none; +} + +.document-layout { + display: grid; + grid-template-columns: minmax(0, 1.35fr) minmax(320px, 0.85fr); + align-items: start; + gap: 22px; +} + +.pdf-section, +.detail-section { + background: var(--panel); + border: 1px solid var(--border); + border-radius: 8px; + padding: 18px; +} + +.pdf-section { + position: sticky; + top: 18px; +} + +.section-heading, +.history-heading { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 12px; +} + +.section-heading h3, +.detail-section h3 { + margin: 0 0 16px; + font-size: 1.05rem; +} + +.section-heading a { + flex-shrink: 0; + color: var(--primary); + font-size: 0.9rem; +} + +#pdf-preview { + display: block; + width: 100%; + height: min(78vh, 1000px); + min-height: 480px; + border: 1px solid var(--border); + background: #e7edf3; +} + +.document-info { + display: grid; + gap: 14px; + min-width: 0; +} + +.metadata-list { + display: grid; + grid-template-columns: minmax(100px, 0.7fr) minmax(0, 1.3fr); + gap: 10px 14px; + margin: 0; + font-size: 0.9rem; +} + +.document-edit-form { + display: grid; + gap: 16px; +} + +.additional-data { + display: grid; + gap: 10px; + margin-top: 18px; + padding-top: 14px; + border-top: 1px solid var(--border); +} + +.additional-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 10px; +} + +.additional-header h4 { + margin: 0; +} + +.additional-field-list { + display: grid; + gap: 10px; +} + +.additional-field-row { + display: grid; + grid-template-columns: minmax(160px, 1fr) minmax(180px, 1.5fr) auto; + gap: 8px; + align-items: center; +} + +.additional-field-row input { + width: 100%; +} + +.additional-field-empty { + margin: 0; + color: var(--muted); + font-style: italic; +} + +.small-btn { + padding: 0.5rem 0.8rem; + font-size: 0.8rem; +} + +.edit-form-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 14px; +} + +.edit-form-grid .field-group input, +.edit-form-grid .field-group select { + width: 100%; + min-width: 0; +} + +.edit-calculated-fields { + display: grid; + gap: 10px; + padding-top: 4px; +} + +.edit-calculated-fields > div { + min-width: 0; +} + +.edit-output-label { + display: block; + margin-bottom: 4px; + color: var(--muted); + font-size: 0.78rem; + font-weight: 600; +} + +.edit-calculated-fields output { + display: block; + overflow-wrap: anywhere; +} + +.edit-form-actions { + display: flex; + align-items: center; + gap: 12px; + flex-wrap: wrap; +} + +.edit-form-actions .action-status { + flex: 1 1 220px; + min-height: 0; + margin: 0; +} + +.metadata-list dt { + color: var(--muted); + font-weight: 600; +} + +.metadata-list dd { + min-width: 0; + margin: 0; + overflow-wrap: anywhere; + white-space: pre-wrap; +} + +.history-list { + display: grid; + gap: 0; + margin: 0; + padding: 0 0 0 18px; +} + +.history-entry, +.history-empty { + padding: 0 0 18px 8px; + border-left: 1px solid var(--border); +} + +.history-entry:last-child, +.history-empty:last-child { + border-left-color: transparent; +} + +.history-heading { + align-items: flex-start; + margin: 0 0 8px; +} + +.history-heading strong { + text-transform: capitalize; +} + +.history-heading time { + color: var(--muted); + font-size: 0.78rem; + text-align: right; +} + +.history-entry pre, +.extracted-section pre { + max-height: 280px; + margin: 0; + overflow: auto; + color: #405365; + font: 0.8rem/1.5 "SFMono-Regular", Consolas, monospace; + white-space: pre-wrap; + overflow-wrap: anywhere; +} + +.extracted-section summary { + cursor: pointer; + font-weight: 650; +} + +.foldable-section > summary { + cursor: pointer; + font-size: 1.05rem; + font-weight: 650; +} + +.foldable-section[open] > summary { + margin-bottom: 16px; +} + +.extracted-section pre { + max-height: 440px; + margin-top: 14px; +} + +@media (max-width: 820px) { + .app-shell { + flex-direction: column; + } + + .toolbar { + align-items: flex-start; + flex-direction: column; + gap: 14px; + } + + .toolbar-actions { + flex-wrap: wrap; + } + + .sidebar { + width: 100%; + } + + .filters-card { + grid-template-columns: 1fr; + } + + .main-panel { + padding: 18px; + } + + .document-layout { + grid-template-columns: minmax(0, 1fr); + } + + .pdf-section { + position: static; + } + + #pdf-preview { + height: 70vh; + min-height: 360px; + } + + .detail-toolbar h2 { + max-width: 100%; + } + + .edit-form-grid { + grid-template-columns: minmax(0, 1fr); + } + + .detail-toolbar-actions { + justify-content: flex-start; + } +}