Document detail
+diff --git a/automatic-pdf-renamer/.dockerignore b/automatic-pdf-renamer/.dockerignore
new file mode 100644
index 0000000..89610b0
--- /dev/null
+++ b/automatic-pdf-renamer/.dockerignore
@@ -0,0 +1,16 @@
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.git/
+.gitignore
+.venv/
+venv/
+.pytest_cache/
+.mypy_cache/
+.DS_Store
+.idea/
+.vscode/
+web/node_modules/
+web/dist/
+*.log
diff --git a/automatic-pdf-renamer/Dockerfile b/automatic-pdf-renamer/Dockerfile
new file mode 100644
index 0000000..600d06e
--- /dev/null
+++ b/automatic-pdf-renamer/Dockerfile
@@ -0,0 +1,24 @@
+FROM python:3.14-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+ PYTHONUNBUFFERED=1 \
+ PIP_NO_CACHE_DIR=1 \
+ APP_HOME=/app \
+ PORT=8000
+
+WORKDIR ${APP_HOME}
+
+COPY . /app
+
+RUN python -m pip install --upgrade pip && \
+ python -m pip install --no-cache-dir \
+ fastapi \
+ uvicorn \
+ boto3 \
+ pymupdf \
+ python-dateutil \
+ PyYAML
+
+EXPOSE 8000
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
diff --git a/automatic-pdf-renamer/app/api/__init__.py b/automatic-pdf-renamer/app/api/__init__.py
new file mode 100644
index 0000000..de5a60b
--- /dev/null
+++ b/automatic-pdf-renamer/app/api/__init__.py
@@ -0,0 +1,5 @@
+"""API package for the Automatic PDF Renamer service."""
+
+from .routes import router
+
+__all__ = ["router"]
diff --git a/automatic-pdf-renamer/app/api/deps.py b/automatic-pdf-renamer/app/api/deps.py
new file mode 100644
index 0000000..a261830
--- /dev/null
+++ b/automatic-pdf-renamer/app/api/deps.py
@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import os
+
+from fastapi import Depends, HTTPException, status
+from fastapi.security import HTTPBasic, HTTPBasicCredentials
+
+security = HTTPBasic(auto_error=False)
+
+
+def get_current_user(
+ credentials: HTTPBasicCredentials | None = Depends(security),
+) -> dict[str, str] | None:
+ """Return the current authenticated user when dashboard auth is configured.
+
+ In v1, auth is optional unless APP_USERNAME and APP_PASSWORD are set. This keeps
+ local development simple while still enabling a basic password gate in production.
+ """
+ username = os.environ.get("APP_USERNAME")
+ password = os.environ.get("APP_PASSWORD")
+
+ if not username and not password:
+ return {"username": "anonymous"}
+
+ if credentials is None:
+ raise HTTPException(
+ status_code=status.HTTP_401_UNAUTHORIZED,
+ detail="Authentication required",
+ headers={"WWW-Authenticate": "Basic realm=automatic-pdf-renamer"},
+ )
+
+ if credentials.username != username or credentials.password != password:
+ raise HTTPException(
+ status_code=status.HTTP_401_UNAUTHORIZED,
+ detail="Invalid username or password",
+ headers={"WWW-Authenticate": "Basic realm=automatic-pdf-renamer"},
+ )
+
+ return {"username": credentials.username}
diff --git a/automatic-pdf-renamer/app/api/models.py b/automatic-pdf-renamer/app/api/models.py
new file mode 100644
index 0000000..aa61961
--- /dev/null
+++ b/automatic-pdf-renamer/app/api/models.py
@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+from datetime import date
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class HealthResponse(BaseModel):
+ """Simple health payload."""
+
+ status: str = "ok"
+ app: str = "automatic-pdf-renamer"
+
+
+class FileListItem(BaseModel):
+ """File metadata returned by the dashboard list endpoint."""
+
+ id: str
+ status: str
+ current_key: str
+ original_key: str | None = None
+ template_name: str | None = None
+ period_format: str | None = None
+ document_type: str | None = None
+ emitting_company: str | None = None
+ period: str | None = None
+ emission_date: str | None = None
+ confidence: float | None = None
+ updated_at: str | None = None
+
+
+class FileDetail(FileListItem):
+ """Detailed file payload."""
+
+ schema_version: str | None = None
+ created_at: str | None = None
+ extracted_text: str | None = None
+ optional_fields: dict[str, Any] = Field(default_factory=dict)
+ history: list[dict[str, Any]] = Field(default_factory=list)
+
+
+class ValidationRequest(BaseModel):
+ """Body for validating or correcting a file."""
+
+ template_id: str | None = None
+ fields: dict[str, Any] = Field(default_factory=dict)
+
+
+class DocumentEditRequest(BaseModel):
+ """Editable document classification and naming fields."""
+
+ template_name: str = Field(min_length=1)
+ period_format: str = Field(min_length=1)
+ emitting_company: str = Field(min_length=1)
+ document_type: str = Field(min_length=1)
+ emission_date: date
+ optional_fields: dict[str, Any] = Field(default_factory=dict)
+
+
+class ValidationResponse(BaseModel):
+ """Validation result payload."""
+
+ id: str
+ status: str
+ current_key: str
+ template_name: str | None = None
+ fields: dict[str, Any] = Field(default_factory=dict)
+ updated: bool = False
diff --git a/automatic-pdf-renamer/app/api/routes.py b/automatic-pdf-renamer/app/api/routes.py
new file mode 100644
index 0000000..f185264
--- /dev/null
+++ b/automatic-pdf-renamer/app/api/routes.py
@@ -0,0 +1,772 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
+
+from app.config import load_templates
+from app.db import FileRecord, SQLiteFileRepository
+from app.period import format_period_from_emission_date
+from app.storage import StorageBackend
+
+from .deps import get_current_user
+from .models import (
+ FileDetail,
+ FileListItem,
+ DocumentEditRequest,
+ HealthResponse,
+ ValidationRequest,
+ ValidationResponse,
+)
+
+router = APIRouter(tags=["files"])
+
+
+def _storage(request: Request) -> StorageBackend:
+ storage = getattr(request.app.state, "storage", None)
+ if storage is None:
+ raise HTTPException(status_code=500, detail="Storage backend is not configured")
+ return storage
+
+
+def _db(request: Request) -> SQLiteFileRepository:
+ db = getattr(request.app.state, "db", None)
+ if db is None:
+ raise HTTPException(status_code=500, detail="SQLite repository is not configured")
+ return db
+
+
+def _registry(request: Request):
+ registry = getattr(request.app.state, "template_registry", None)
+ if registry is None:
+ registry = load_templates()
+ request.app.state.template_registry = registry
+ return registry
+
+
+def _parse_file_metadata(raw: str | bytes | None) -> dict[str, Any]:
+ if raw is None:
+ return {}
+ if isinstance(raw, bytes):
+ raw = raw.decode("utf-8", errors="replace")
+ try:
+ return json.loads(raw)
+ except json.JSONDecodeError:
+ return {}
+
+
+def _as_file_item(file_id: str, metadata: dict[str, Any]) -> FileListItem:
+ return FileListItem(
+ id=file_id,
+ status=str(metadata.get("status", "unknown")),
+ current_key=str(metadata.get("current_key", metadata.get("key", ""))),
+ original_key=metadata.get("original_key"),
+ template_name=metadata.get("template_name"),
+ period_format=metadata.get("period_format"),
+ document_type=metadata.get("document_type"),
+ emitting_company=metadata.get("emitting_company"),
+ period=metadata.get("period"),
+ emission_date=metadata.get("emission_date"),
+ confidence=metadata.get("confidence"),
+ updated_at=metadata.get("updated_at"),
+ )
+
+
+@router.get("/health", response_model=HealthResponse)
+async def health() -> HealthResponse:
+ """Service health checkpoint."""
+ return HealthResponse(status="ok", app="automatic-pdf-renamer")
+
+
+@router.get("/files", response_model=list[FileListItem])
+async def list_files(
+ request: Request,
+ response: Response,
+ current_user: dict[str, str] | None = Depends(get_current_user),
+ status: str | None = None,
+ template: str | None = None,
+ period_format: str | None = None,
+ q: str | None = None,
+ current_key: str | None = None,
+ emitting_company: str | None = None,
+ document_type: str | None = None,
+ period: str | None = None,
+ confidence: str | None = None,
+ updated_at: str | None = None,
+ sort_by: str = "updated_at",
+ sort_order: str = "desc",
+ page: int = 1,
+) -> list[FileListItem]:
+ """List documents and report filtered and global counts in response headers."""
+ storage = _storage(request)
+ db = _db(request)
+ filters = {
+ "status": status,
+ "template": template,
+ "q": q,
+ "current_key": current_key,
+ "emitting_company": emitting_company,
+ "document_type": document_type,
+ "period": period,
+ "period_format": period_format,
+ "confidence": confidence,
+ "updated_at": updated_at,
+ }
+
+ if db.count_files() == 0:
+ for key in sorted(storage.list_objects(prefix="")):
+ if not key.endswith(".meta.json"):
+ continue
+ metadata = _parse_file_metadata(storage.get_object(key))
+ if metadata:
+ db.upsert(FileRecord.from_metadata(metadata))
+
+ records = db.list_files(
+ **filters,
+ sort_by=sort_by,
+ sort_order=sort_order,
+ page=page,
+ page_size=25,
+ )
+ response.headers["X-Total-Count"] = str(db.count_files(**filters))
+ response.headers["X-Status-Counts"] = json.dumps(db.get_status_counts())
+
+ return [
+ FileListItem(
+ id=record.id,
+ status=record.status,
+ current_key=record.current_key,
+ original_key=record.original_key,
+ template_name=record.template_name,
+ period_format=record.period_format,
+ document_type=record.document_type,
+ emitting_company=record.emitting_company,
+ period=record.period,
+ emission_date=record.emission_date,
+ confidence=record.confidence,
+ updated_at=record.updated_at,
+ )
+ for record in records
+ ]
+
+
+@router.get("/files/{file_id}", response_model=FileDetail)
+async def get_file(
+ request: Request,
+ file_id: str,
+ current_user: dict[str, str] | None = Depends(get_current_user),
+) -> FileDetail:
+ """Return metadata and extracted text for a single file."""
+ storage = _storage(request)
+ db = _db(request)
+ record = db.get_by_id(file_id)
+
+ if record is None:
+ keys = storage.list_objects(prefix="")
+ for key in keys:
+ if not key.endswith(".meta.json"):
+ continue
+
+ metadata = _parse_file_metadata(storage.get_object(key))
+ if metadata and (metadata.get("sha256") == file_id or key == file_id):
+ record = FileRecord.from_metadata(metadata)
+ db.upsert(record)
+ break
+
+ if record is None:
+ raise HTTPException(status_code=404, detail=f"File '{file_id}' not found")
+
+ sidecar_metadata: dict[str, Any] = {}
+ if record.current_key:
+ sidecar_metadata = _parse_file_metadata(
+ storage.get_object(f"{record.current_key}.meta.json")
+ )
+ if not sidecar_metadata or sidecar_metadata.get("sha256") not in {
+ record.sha256,
+ record.id,
+ }:
+ for key in storage.list_objects(prefix=""):
+ if not key.endswith(".meta.json"):
+ continue
+ candidate = _parse_file_metadata(storage.get_object(key))
+ if candidate.get("sha256") in {record.sha256, record.id}:
+ sidecar_metadata = candidate
+ break
+
+ metadata = {
+ "schema_version": sidecar_metadata.get("schema_version"),
+ "sha256": record.sha256,
+ "status": record.status,
+ "current_key": record.current_key,
+ "original_key": record.original_key,
+ "template_name": record.template_name,
+ "period_format": record.period_format,
+ "document_type": record.document_type,
+ "emitting_company": record.emitting_company,
+ "period": record.period,
+ "emission_date": record.emission_date,
+ "confidence": record.confidence,
+ "updated_at": record.updated_at,
+ "created_at": record.created_at,
+ "extracted_text": record.extracted_text,
+ "optional_fields": record.optional_fields,
+ "history": sidecar_metadata.get("events", []),
+ }
+ item = _as_file_item(record.id, metadata)
+ return FileDetail(
+ id=item.id,
+ status=item.status,
+ current_key=item.current_key,
+ original_key=item.original_key,
+ template_name=item.template_name,
+ period_format=item.period_format,
+ document_type=item.document_type,
+ emitting_company=item.emitting_company,
+ period=item.period,
+ emission_date=item.emission_date,
+ confidence=item.confidence,
+ updated_at=item.updated_at,
+ schema_version=metadata["schema_version"],
+ created_at=metadata["created_at"],
+ extracted_text=record.extracted_text,
+ optional_fields=record.optional_fields,
+ history=metadata["history"] if isinstance(metadata["history"], list) else [],
+ )
+
+
+@router.get("/files/{file_id}/pdf")
+async def get_file_pdf(
+ request: Request,
+ file_id: str,
+ current_user: dict[str, str] | None = Depends(get_current_user),
+) -> Response:
+ """Stream a PDF by key from storage when present."""
+ storage = _storage(request)
+ keys = storage.list_objects(prefix="")
+
+ for key in keys:
+ if key.endswith(".meta.json"):
+ metadata = _parse_file_metadata(storage.get_object(key))
+ if metadata and (metadata.get("sha256") == file_id or key == file_id):
+ current_key = metadata.get("current_key") or key.replace(".meta.json", "")
+ pdf_keys = [current_key]
+ if not current_key.lower().endswith(".pdf"):
+ pdf_keys.append(f"{current_key}.pdf")
+ for pdf_key in pdf_keys:
+ payload = storage.get_object(pdf_key)
+ if payload is not None:
+ return Response(content=payload, media_type="application/pdf")
+ raise HTTPException(status_code=404, detail="PDF bytes not found for the file")
+
+ raise HTTPException(status_code=404, detail=f"PDF for '{file_id}' not found")
+
+
+@router.post("/files/{file_id}/edit", response_model=FileDetail)
+async def edit_file(
+ request: Request,
+ file_id: str,
+ payload: DocumentEditRequest,
+ current_user: dict[str, str] | None = Depends(get_current_user),
+) -> FileDetail:
+ """Recalculate and persist user-edited document metadata and its storage key."""
+ storage = _storage(request)
+ db = _db(request)
+ registry = _registry(request)
+ record = db.get_by_id(file_id)
+ if record is None:
+ raise HTTPException(status_code=404, detail=f"File '{file_id}' not found")
+
+ template_name = payload.template_name.strip()
+ period_format = payload.period_format.strip()
+ emitting_company = payload.emitting_company.strip()
+ document_type = payload.document_type.strip()
+ if not emitting_company or not document_type:
+ raise HTTPException(status_code=422, detail="Company and document type are required")
+
+ if registry.get_template(template_name) is None:
+ raise HTTPException(status_code=422, detail=f"Unknown template '{template_name}'")
+ period_config = registry.get_period_format_config(period_format)
+ if period_config is None:
+ raise HTTPException(status_code=422, detail=f"Unknown period format '{period_format}'")
+
+ emission_date = payload.emission_date.isoformat()
+ try:
+ period = format_period_from_emission_date(
+ payload.emission_date,
+ period_config.to_dict(),
+ )
+ except (TypeError, ValueError) as error:
+ raise HTTPException(status_code=422, detail=f"Unable to calculate period: {error}") from error
+
+ updated_optional_fields = {
+ str(label).strip(): value
+ for label, value in (payload.optional_fields or {}).items()
+ if str(label).strip()
+ }
+
+ key_pattern = registry.get_key_pattern(template_name)
+ if not key_pattern:
+ raise HTTPException(status_code=422, detail=f"Template '{template_name}' has no key pattern")
+ pattern_values = dict(updated_optional_fields)
+ pattern_values.update({
+ "company": emitting_company,
+ "document_type": document_type,
+ "emission_date": emission_date,
+ "period": period,
+ "period_format": period_format,
+ "template_name": template_name,
+ "emitting_company": emitting_company,
+ "original_filename": record.original_key.rsplit("/", 1)[-1],
+ })
+ for _ in range(20):
+ try:
+ formatted_key = key_pattern.format(**pattern_values).strip("/")
+ break
+ except KeyError as error:
+ pattern_values[str(error.args[0])] = "unknown"
+ else:
+ raise HTTPException(status_code=422, detail="Template key pattern has too many unresolved fields")
+
+ if not formatted_key or any(part in {".", ".."} for part in formatted_key.split("/")):
+ raise HTTPException(status_code=422, detail="Template produced an invalid document key")
+ new_current_key = formatted_key if formatted_key.startswith("files/") else f"files/{formatted_key}"
+ new_pdf_key = new_current_key if new_current_key.lower().endswith(".pdf") else f"{new_current_key}.pdf"
+ new_sidecar_key = f"{new_current_key}.meta.json"
+
+ source_sidecar_key = ""
+ sidecar_metadata: dict[str, Any] = {}
+ known_sidecar_key = f"{record.current_key}.meta.json" if record.current_key else ""
+ if known_sidecar_key:
+ candidate = _parse_file_metadata(storage.get_object(known_sidecar_key))
+ if candidate and candidate.get("sha256") in {record.sha256, record.id}:
+ source_sidecar_key = known_sidecar_key
+ sidecar_metadata = candidate
+ if not source_sidecar_key:
+ for key in storage.list_objects(prefix=""):
+ if not key.endswith(".meta.json"):
+ continue
+ candidate = _parse_file_metadata(storage.get_object(key))
+ if candidate and (
+ candidate.get("sha256") in {record.sha256, record.id}
+ or candidate.get("current_key") == record.current_key
+ ):
+ source_sidecar_key = key
+ sidecar_metadata = candidate
+ break
+ if not source_sidecar_key:
+ raise HTTPException(status_code=404, detail="Document sidecar was not found")
+
+ source_current_key = str(sidecar_metadata.get("current_key") or record.current_key)
+ source_pdf_key = ""
+ pdf_candidates = [source_current_key]
+ if source_current_key and not source_current_key.lower().endswith(".pdf"):
+ pdf_candidates.append(f"{source_current_key}.pdf")
+ for candidate in pdf_candidates:
+ if candidate and storage.object_exists(candidate):
+ source_pdf_key = candidate
+ break
+
+ if storage.object_exists(new_pdf_key) and new_pdf_key != source_pdf_key:
+ raise HTTPException(status_code=409, detail="The recalculated PDF key is already in use")
+ destination_sidecar = _parse_file_metadata(storage.get_object(new_sidecar_key))
+ if (
+ destination_sidecar
+ and new_sidecar_key != source_sidecar_key
+ and destination_sidecar.get("sha256") not in {record.sha256, record.id}
+ ):
+ raise HTTPException(status_code=409, detail="The recalculated sidecar key is already in use")
+
+ updated_at = datetime.now(timezone.utc).isoformat()
+ old_values = {}
+ new_values = {}
+ if template_name != record.template_name:
+ old_values["template_name"] = record.template_name
+ new_values["template_name"] = template_name
+ if period_format != record.period_format:
+ old_values["period_format"] = record.period_format
+ new_values["period_format"] = period_format
+ if emitting_company != record.emitting_company:
+ old_values["emitting_company"] = record.emitting_company
+ new_values["emitting_company"] = emitting_company
+ if document_type != record.document_type:
+ old_values["document_type"] = record.document_type
+ new_values["document_type"] = document_type
+ if emission_date != record.emission_date:
+ old_values["emission_date"] = record.emission_date
+ new_values["emission_date"] = emission_date
+ if period != record.period:
+ old_values["period"] = record.period
+ new_values["period"] = period
+ if new_pdf_key != record.current_key:
+ old_values["current_key"] = record.current_key
+ new_values["current_key"] = new_pdf_key
+ if updated_optional_fields != record.optional_fields:
+ old_values["optional_fields"] = dict(record.optional_fields)
+ new_values["optional_fields"] = dict(updated_optional_fields)
+ sidecar_metadata.update(new_values)
+ sidecar_metadata["optional_fields"] = updated_optional_fields
+ sidecar_metadata["status"] = "validated"
+ sidecar_metadata["updated_at"] = updated_at
+ events = sidecar_metadata.get("events")
+ if not isinstance(events, list):
+ events = []
+ sidecar_metadata["events"] = events
+ events.append({
+ "type": "corrected",
+ "timestamp": updated_at,
+ "payload": {"before": old_values, "after": new_values},
+ })
+
+ copied_pdf = bool(source_pdf_key and source_pdf_key != new_pdf_key)
+ if copied_pdf:
+ storage.copy_object(source_pdf_key, new_pdf_key)
+ try:
+ storage.put_object(new_sidecar_key, json.dumps(sidecar_metadata, ensure_ascii=False, indent=2))
+ except Exception:
+ if copied_pdf:
+ storage.delete_object(new_pdf_key)
+ raise
+
+ record.template_name = template_name
+ record.period_format = period_format
+ record.emitting_company = emitting_company
+ record.document_type = document_type
+ record.emission_date = emission_date
+ record.period = period
+ record.current_key = new_pdf_key
+ record.optional_fields = updated_optional_fields
+ record.status = "validated"
+ record.updated_at = updated_at
+ db.upsert(record)
+
+ if source_sidecar_key != new_sidecar_key:
+ storage.delete_object(source_sidecar_key)
+ if source_pdf_key and source_pdf_key != new_pdf_key:
+ storage.delete_object(source_pdf_key)
+
+ return await get_file(request, record.id, current_user=current_user)
+
+
+@router.post("/files/{file_id}/validate", response_model=ValidationResponse)
+async def validate_file(
+ request: Request,
+ file_id: str,
+ payload: ValidationRequest,
+ current_user: dict[str, str] | None = Depends(get_current_user),
+) -> ValidationResponse:
+ """Apply a manual validation update and persist it back to the sidecar metadata."""
+ storage = _storage(request)
+ db = _db(request)
+ record = db.get_by_id(file_id)
+
+ if record is None:
+ keys = storage.list_objects(prefix="")
+ for key in keys:
+ if key.endswith(".meta.json"):
+ metadata = _parse_file_metadata(storage.get_object(key))
+ if metadata and (metadata.get("sha256") == file_id or key == file_id):
+ record = FileRecord.from_metadata(metadata)
+ db.upsert(record)
+ break
+
+ if record is None:
+ raise HTTPException(status_code=404, detail=f"File '{file_id}' not found")
+
+ metadata = dict(record.optional_fields)
+ if payload.template_id:
+ record.template_name = payload.template_id
+ if payload.fields:
+ record.optional_fields = {**record.optional_fields, **payload.fields}
+ metadata = record.optional_fields
+
+ record.status = "validated"
+ record.updated_at = "now"
+ db.upsert(record)
+
+ sidecar_key = f"{record.current_key}.meta.json" if record.current_key else f"{file_id}.meta.json"
+ storage.put_object(sidecar_key, json.dumps({
+ "sha256": record.sha256,
+ "original_key": record.original_key,
+ "current_key": record.current_key,
+ "status": record.status,
+ "template_name": record.template_name,
+ "period_format": record.period_format,
+ "document_type": record.document_type,
+ "emitting_company": record.emitting_company,
+ "period": record.period,
+ "emission_date": record.emission_date,
+ "confidence": record.confidence,
+ "optional_fields": record.optional_fields,
+ "extracted_text": record.extracted_text,
+ "updated_at": record.updated_at,
+ }, ensure_ascii=False, indent=2))
+
+ return ValidationResponse(
+ id=str(record.sha256 or file_id),
+ status="validated",
+ current_key=record.current_key,
+ template_name=record.template_name,
+ fields=record.optional_fields,
+ updated=True,
+ )
+
+
+@router.get("/api/document-options")
+async def document_edit_options(
+ request: Request,
+ current_user: dict[str, str] | None = Depends(get_current_user),
+) -> dict[str, list[str]]:
+ """Return configured templates and existing document values for edit suggestions."""
+ registry = _registry(request)
+ db = _db(request)
+ records = db.list_files(page_size=10000)
+
+ companies = sorted({
+ str(record.emitting_company).strip()
+ for record in records
+ if record.emitting_company and str(record.emitting_company).strip()
+ })
+ document_types = sorted({
+ str(record.document_type).strip()
+ for record in records
+ if record.document_type and str(record.document_type).strip()
+ })
+ optional_field_names = sorted({
+ str(name).strip()
+ for record in records
+ for name in (record.optional_fields or {}).keys()
+ if str(name).strip()
+ })
+ optional_field_values = sorted({
+ str(value).strip()
+ for record in records
+ for value in (record.optional_fields or {}).values()
+ if value is not None and str(value).strip()
+ })
+
+ return {
+ "templates": registry.list_all_templates(),
+ "period_formats": registry.list_period_formats(),
+ "companies": companies,
+ "document_types": document_types,
+ "optional_field_names": optional_field_names,
+ "optional_field_values": optional_field_values,
+ }
+
+
+@router.get("/templates")
+async def list_templates(
+ request: Request,
+ current_user: dict[str, str] | None = Depends(get_current_user),
+) -> list[dict[str, Any]]:
+ """List all templates configured for the service."""
+ registry = _registry(request)
+ items: list[dict[str, Any]] = []
+ for template_name in registry.list_all_templates():
+ template = registry.get_template(template_name)
+ if template is None:
+ continue
+ items.append({
+ "name": template.name,
+ "key_pattern": template.key_pattern,
+ })
+ return items
+
+
+@router.post("/retrain")
+async def start_retrain(
+ request: Request,
+ current_user: dict[str, str] | None = Depends(get_current_user),
+) -> dict[str, Any]:
+ """Start a retraining job. The current implementation records a job stub and returns its status."""
+ request.app.state.last_retrain_job = {
+ "job_id": "retrain-1",
+ "status": "queued",
+ "message": "Retrain job accepted. Replace with a subprocess-backed job in production.",
+ }
+ return request.app.state.last_retrain_job
+
+
+@router.get("/retrain/status")
+async def get_retrain_status(
+ request: Request,
+ current_user: dict[str, str] | None = Depends(get_current_user),
+) -> dict[str, Any]:
+ """Read the active retraining status."""
+ job = getattr(request.app.state, "last_retrain_job", {"job_id": "retrain-1", "status": "idle"})
+ return job
+
+
+@router.post("/retrain/{job_id}/activate")
+async def activate_model(
+ job_id: str,
+ current_user: dict[str, str] | None = Depends(get_current_user),
+) -> dict[str, Any]:
+ """Activate a completed training model."""
+ return {"job_id": job_id, "status": "activated", "message": "Model activation accepted."}
+
+
+@router.get("/consistency")
+async def list_consistency_flags(
+ request: Request,
+ current_user: dict[str, str] | None = Depends(get_current_user),
+) -> list[dict[str, Any]]:
+ """Return consistency anomalies. In the current implementation, this is a lightweight placeholder."""
+ storage = _storage(request)
+ db = _db(request)
+ anomalies: list[dict[str, Any]] = []
+ for key in storage.list_objects(prefix=""):
+ if key.endswith(".meta.json"):
+ metadata = _parse_file_metadata(storage.get_object(key))
+ if metadata and metadata.get("status") in {"processed", "validated"}:
+ current_key = metadata.get("current_key", key.replace(".meta.json", ""))
+ if not storage.object_exists(current_key):
+ anomalies.append({
+ "kind": "missing_object",
+ "key": key,
+ "message": "Sidecar points to an object that is no longer present.",
+ })
+ for record in db.list_files():
+ if record.status in {"processed", "validated"} and not storage.object_exists(record.current_key):
+ anomalies.append({
+ "kind": "missing_object",
+ "key": record.current_key,
+ "message": "SQLite record points to a missing object.",
+ })
+ return anomalies
+
+
+@router.post("/consistency/rebuild")
+async def rebuild_consistency(
+ request: Request,
+ current_user: dict[str, str] | None = Depends(get_current_user),
+) -> dict[str, Any]:
+ """Rebuild SQLite from sidecar metadata and return the completed result."""
+ storage = _storage(request)
+ db = _db(request)
+ rebuilt = db.rebuild_from_sidecars(storage)
+ request.app.state.last_rebuild = {
+ "status": "completed",
+ "message": "SQLite rebuilt from sidecar metadata.",
+ "count": len(rebuilt),
+ }
+ return request.app.state.last_rebuild
+
+
+@router.get("/api/health")
+async def api_health() -> HealthResponse:
+ """Alias for /health."""
+ return await health()
+
+
+@router.get("/api/files", response_model=list[FileListItem])
+async def list_files_api(
+ request: Request,
+ response: Response,
+ current_user: dict[str, str] | None = Depends(get_current_user),
+ status: str | None = None,
+ template: str | None = None,
+ period_format: str | None = None,
+ q: str | None = None,
+ current_key: str | None = None,
+ emitting_company: str | None = None,
+ document_type: str | None = None,
+ period: str | None = None,
+ confidence: str | None = None,
+ updated_at: str | None = None,
+ sort_by: str = "updated_at",
+ sort_order: str = "desc",
+ page: int = 1,
+) -> list[FileListItem]:
+ """Compatibility alias for /files."""
+ return await list_files(
+ request=request,
+ response=response,
+ current_user=current_user,
+ status=status,
+ template=template,
+ period_format=period_format,
+ q=q,
+ current_key=current_key,
+ emitting_company=emitting_company,
+ document_type=document_type,
+ period=period,
+ confidence=confidence,
+ updated_at=updated_at,
+ sort_by=sort_by,
+ sort_order=sort_order,
+ page=page,
+ )
+
+
+@router.get("/api/files/{file_id}", response_model=FileDetail)
+async def get_file_api(
+ request: Request,
+ file_id: str,
+ current_user: dict[str, str] | None = Depends(get_current_user),
+) -> FileDetail:
+ """Compatibility alias for /files/{file_id}."""
+ return await get_file(request=request, file_id=file_id, current_user=current_user)
+
+
+@router.post("/api/files/{file_id}/edit", response_model=FileDetail)
+async def edit_file_api(
+ request: Request,
+ file_id: str,
+ payload: DocumentEditRequest,
+ current_user: dict[str, str] | None = Depends(get_current_user),
+) -> FileDetail:
+ """Compatibility alias for POST /files/{file_id}/edit."""
+ return await edit_file(
+ request=request,
+ file_id=file_id,
+ payload=payload,
+ current_user=current_user,
+ )
+
+
+@router.get("/api/templates")
+async def list_templates_api(
+ request: Request,
+ current_user: dict[str, str] | None = Depends(get_current_user),
+) -> list[dict[str, Any]]:
+ """Compatibility alias for /templates."""
+ return await list_templates(request=request, current_user=current_user)
+
+
+@router.post("/api/retrain")
+async def start_retrain_api(
+ request: Request,
+ current_user: dict[str, str] | None = Depends(get_current_user),
+) -> dict[str, Any]:
+ """Compatibility alias for /retrain."""
+ return await start_retrain(request=request, current_user=current_user)
+
+
+@router.get("/api/retrain/status")
+async def get_retrain_status_api(
+ request: Request,
+ current_user: dict[str, str] | None = Depends(get_current_user),
+) -> dict[str, Any]:
+ """Compatibility alias for /retrain/status."""
+ return await get_retrain_status(request=request, current_user=current_user)
+
+
+@router.get("/api/consistency")
+async def list_consistency_flags_api(
+ request: Request,
+ current_user: dict[str, str] | None = Depends(get_current_user),
+) -> list[dict[str, Any]]:
+ """Compatibility alias for /consistency."""
+ return await list_consistency_flags(request=request, current_user=current_user)
+
+
+@router.post("/api/consistency/rebuild")
+async def rebuild_consistency_api(
+ request: Request,
+ current_user: dict[str, str] | None = Depends(get_current_user),
+) -> dict[str, Any]:
+ """Compatibility alias for /consistency/rebuild."""
+ return await rebuild_consistency(request=request, current_user=current_user)
diff --git a/automatic-pdf-renamer/app/config.py b/automatic-pdf-renamer/app/config.py
index cb86c15..5570dca 100644
--- a/automatic-pdf-renamer/app/config.py
+++ b/automatic-pdf-renamer/app/config.py
@@ -60,31 +60,6 @@ class S3Config:
raise ValueError("S3_BUCKET_NAME environment variable is required")
-@dataclass
-class PaperlessConfig:
- """Paperless-ngx API configuration."""
-
- url: str
- token: str
- timeout: int = 30
-
- @classmethod
- def from_env(cls) -> "PaperlessConfig":
- """Create PaperlessConfig from environment variables."""
- return cls(
- url=os.environ.get("PAPERLESS_URL", ""),
- token=os.environ.get("PAPERLESS_TOKEN", ""),
- timeout=int(os.environ.get("PAPERLESS_TIMEOUT", "30")),
- )
-
- def validate(self) -> None:
- """Validate that all required fields are set."""
- if not self.url:
- raise ValueError("PAPERLESS_URL environment variable is required")
- if not self.token:
- raise ValueError("PAPERLESS_TOKEN environment variable is required")
-
-
@dataclass
class PollerConfig:
"""Inbox poller configuration."""
@@ -268,8 +243,9 @@ class TemplateConfig:
class AppConfig:
"""Main application configuration."""
+ storage_backend: str = "s3"
+ local_storage_dir: str = "./storage"
s3: S3Config = field(default_factory=S3Config)
- paperless: PaperlessConfig | None = None
poller: PollerConfig = field(default_factory=PollerConfig)
classifier: ClassifierConfig = field(default_factory=ClassifierConfig)
debug: bool = False
@@ -279,27 +255,32 @@ class AppConfig:
def from_env(cls) -> "AppConfig":
"""Create AppConfig from environment variables."""
debug = os.environ.get("DEBUG", "").lower() in ("true", "1", "yes")
+ storage_backend = os.environ.get("APP_STORAGE_BACKEND", os.environ.get("STORAGE_BACKEND", "s3")).lower()
config = cls(
+ storage_backend=storage_backend,
+ local_storage_dir=os.environ.get("LOCAL_STORAGE_DIR", "./storage"),
s3=S3Config.from_env(),
- paperless=None, # Only needed for import script
poller=PollerConfig.from_env(),
classifier=ClassifierConfig.from_env(),
debug=debug,
sqlite_path=os.environ.get("SQLITE_PATH", "./renamer.db"),
)
- # Only load paperless config if PAPERLESS_URL is set
- if os.environ.get("PAPERLESS_URL"):
- config.paperless = PaperlessConfig.from_env()
+ if config.storage_backend not in {"s3", "file"}:
+ raise ValueError("STORAGE_BACKEND must be either 's3' or 'file'")
return config
def validate(self) -> None:
"""Validate the configuration."""
- self.s3.validate()
- if self.paperless:
- self.paperless.validate()
+ if self.storage_backend == "s3":
+ self.s3.validate()
+ elif self.storage_backend == "file":
+ if not self.local_storage_dir:
+ raise ValueError("LOCAL_STORAGE_DIR is required when STORAGE_BACKEND=file")
+ else:
+ raise ValueError("STORAGE_BACKEND must be either 's3' or 'file'")
class TemplateRegistry:
@@ -617,6 +598,10 @@ class TemplateRegistry:
"""
return self._period_formats.get(format_name)
+ def list_period_formats(self) -> list[str]:
+ """List configured named period formats."""
+ return list(self._period_formats.keys())
+
def get_key_pattern(self, template_name: str) -> str | None:
"""Get the key pattern for a template.
diff --git a/automatic-pdf-renamer/app/db.py b/automatic-pdf-renamer/app/db.py
new file mode 100644
index 0000000..6bac280
--- /dev/null
+++ b/automatic-pdf-renamer/app/db.py
@@ -0,0 +1,417 @@
+from __future__ import annotations
+
+import json
+import logging
+import os
+import sqlite3
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger("uvicorn.error")
+
+
+@dataclass
+class FileRecord:
+ """SQLite-backed record for a document file."""
+
+ id: str
+ sha256: str
+ original_key: str
+ current_key: str
+ status: str = "unknown"
+ template_name: str | None = None
+ period_format: str | None = None
+ document_type: str | None = None
+ emitting_company: str | None = None
+ period: str | None = None
+ emission_date: str | None = None
+ confidence: float | None = None
+ optional_fields: dict[str, Any] = field(default_factory=dict)
+ extracted_text: str | None = None
+ created_at: str = ""
+ updated_at: str = ""
+
+ @classmethod
+ def from_metadata(cls, metadata: dict[str, Any]) -> "FileRecord":
+ record_id = str(metadata.get("sha256") or metadata.get("id") or metadata.get("current_key") or "unknown")
+ created_at = metadata.get("created_at") or metadata.get("created") or _utc_now()
+ updated_at = metadata.get("updated_at") or created_at
+ return cls(
+ id=record_id,
+ sha256=str(metadata.get("sha256") or record_id),
+ original_key=str(metadata.get("original_key") or metadata.get("source_key") or ""),
+ current_key=str(metadata.get("current_key") or metadata.get("key") or ""),
+ status=str(metadata.get("status") or "unknown"),
+ template_name=metadata.get("template_name"),
+ period_format=metadata.get("period_format"),
+ document_type=metadata.get("document_type"),
+ emitting_company=metadata.get("emitting_company"),
+ period=metadata.get("period"),
+ emission_date=metadata.get("emission_date"),
+ confidence=metadata.get("confidence"),
+ optional_fields=dict(metadata.get("optional_fields") or {}),
+ extracted_text=metadata.get("extracted_text"),
+ created_at=created_at,
+ updated_at=updated_at,
+ )
+
+ def to_row(self) -> tuple[Any, ...]:
+ return (
+ self.id,
+ self.sha256,
+ self.original_key,
+ self.current_key,
+ self.status,
+ self.template_name,
+ self.period_format,
+ self.document_type,
+ self.emitting_company,
+ self.period,
+ self.emission_date,
+ self.confidence,
+ json.dumps(self.optional_fields, ensure_ascii=False),
+ self.extracted_text,
+ self.created_at or _utc_now(),
+ self.updated_at or self.created_at or _utc_now(),
+ )
+
+ @classmethod
+ def from_row(cls, row: tuple[Any, ...]) -> "FileRecord":
+ (
+ id_,
+ sha256,
+ original_key,
+ current_key,
+ status,
+ template_name,
+ period_format,
+ document_type,
+ emitting_company,
+ period,
+ emission_date,
+ confidence,
+ optional_fields,
+ extracted_text,
+ created_at,
+ updated_at,
+ ) = row
+ try:
+ optional_value = json.loads(optional_fields or "{}") if optional_fields else {}
+ except json.JSONDecodeError:
+ optional_value = {}
+
+ return cls(
+ id=str(id_),
+ sha256=str(sha256),
+ original_key=str(original_key),
+ current_key=str(current_key),
+ status=str(status),
+ template_name=template_name,
+ period_format=period_format,
+ document_type=document_type,
+ emitting_company=emitting_company,
+ period=period,
+ emission_date=emission_date,
+ confidence=float(confidence) if confidence is not None else None,
+ optional_fields=dict(optional_value),
+ extracted_text=extracted_text,
+ created_at=str(created_at),
+ updated_at=str(updated_at),
+ )
+
+
+def _utc_now() -> str:
+ return datetime.now(timezone.utc).isoformat()
+
+
+class SQLiteFileRepository:
+ """Repository for storing and querying file metadata in SQLite."""
+
+ def __init__(self, db_path: str | None = None):
+ default_path = db_path or os.environ.get("SQLITE_PATH", "./renamer.db")
+ self.db_path = str(Path(default_path).resolve())
+ self._ensure_parent_dir()
+ self.init_db()
+
+ def _ensure_parent_dir(self) -> None:
+ directory = Path(self.db_path).parent
+ directory.mkdir(parents=True, exist_ok=True)
+
+ def _connect(self) -> sqlite3.Connection:
+ conn = sqlite3.connect(self.db_path)
+ conn.row_factory = sqlite3.Row
+ return conn
+
+ def init_db(self) -> None:
+ with self._connect() as conn:
+ conn.execute(
+ """
+ CREATE TABLE IF NOT EXISTS files (
+ id TEXT PRIMARY KEY,
+ sha256 TEXT NOT NULL UNIQUE,
+ original_key TEXT,
+ current_key TEXT,
+ status TEXT,
+ template_name TEXT,
+ period_format TEXT,
+ document_type TEXT,
+ emitting_company TEXT,
+ period TEXT,
+ emission_date TEXT,
+ confidence REAL,
+ optional_fields TEXT,
+ extracted_text TEXT,
+ created_at TEXT,
+ updated_at TEXT
+ )
+ """
+ )
+ columns = {row[1] for row in conn.execute("PRAGMA table_info(files)")}
+ if "period_format" not in columns:
+ conn.execute("ALTER TABLE files ADD COLUMN period_format TEXT")
+ conn.execute(
+ """
+ CREATE INDEX IF NOT EXISTS idx_files_status
+ ON files(status)
+ """
+ )
+ conn.execute(
+ """
+ CREATE INDEX IF NOT EXISTS idx_files_template
+ ON files(template_name)
+ """
+ )
+ conn.execute(
+ """
+ CREATE INDEX IF NOT EXISTS idx_files_period_format
+ ON files(period_format)
+ """
+ )
+ conn.commit()
+
+ def upsert(self, record: FileRecord) -> FileRecord:
+ record.updated_at = record.updated_at or _utc_now()
+ record.created_at = record.created_at or record.updated_at
+
+ with self._connect() as conn:
+ conn.execute(
+ """
+ INSERT INTO files (
+ id, sha256, original_key, current_key, status, template_name, period_format,
+ document_type, emitting_company, period, emission_date,
+ confidence, optional_fields, extracted_text, created_at, updated_at
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT(id) DO UPDATE SET
+ sha256 = excluded.sha256,
+ original_key = excluded.original_key,
+ current_key = excluded.current_key,
+ status = excluded.status,
+ template_name = excluded.template_name,
+ period_format = excluded.period_format,
+ document_type = excluded.document_type,
+ emitting_company = excluded.emitting_company,
+ period = excluded.period,
+ emission_date = excluded.emission_date,
+ confidence = excluded.confidence,
+ optional_fields = excluded.optional_fields,
+ extracted_text = excluded.extracted_text,
+ updated_at = excluded.updated_at
+ """,
+ record.to_row(),
+ )
+ conn.commit()
+ return record
+
+ def get_by_id(self, file_id: str) -> FileRecord | None:
+ with self._connect() as conn:
+ row = conn.execute(
+ "SELECT * FROM files WHERE id = ? OR sha256 = ?",
+ (file_id, file_id),
+ ).fetchone()
+ if row is None:
+ return None
+ return FileRecord.from_row(tuple(row))
+
+ def _file_conditions(
+ self,
+ status: str | None = None,
+ template: str | None = None,
+ period_format: str | None = None,
+ q: str | None = None,
+ current_key: str | None = None,
+ emitting_company: str | None = None,
+ document_type: str | None = None,
+ period: str | None = None,
+ confidence: str | None = None,
+ updated_at: str | None = None,
+ ) -> tuple[list[str], list[Any]]:
+ conditions: list[str] = []
+ params: list[Any] = []
+
+ if status:
+ conditions.append("status = ?")
+ params.append(status)
+ if template:
+ conditions.append("LOWER(template_name) LIKE ?")
+ params.append(f"%{template.lower()}%")
+ if current_key:
+ value = f"%{current_key.lower()}%"
+ conditions.append("(LOWER(current_key) LIKE ? OR LOWER(template_name) LIKE ?)")
+ params.extend([value, value])
+ for column, value in (
+ ("emitting_company", emitting_company),
+ ("document_type", document_type),
+ ("period", period),
+ ("updated_at", updated_at),
+ ("period_format", period_format),
+ ):
+ if value:
+ conditions.append(f"LOWER({column}) LIKE ?")
+ params.append(f"%{value.lower()}%")
+ if confidence:
+ confidence_filter = confidence.strip().removesuffix("%")
+ if confidence_filter:
+ conditions.append("CAST(ROUND(confidence * 100, 1) AS TEXT) LIKE ?")
+ params.append(f"%{confidence_filter.lower()}%")
+ if q:
+ conditions.append(
+ "(LOWER(current_key) LIKE ? OR LOWER(original_key) LIKE ? OR LOWER(template_name) LIKE ? OR LOWER(document_type) LIKE ? OR LOWER(emitting_company) LIKE ? OR LOWER(period) LIKE ? OR LOWER(emission_date) LIKE ? OR LOWER(period_format) LIKE ?)"
+ )
+ like_value = f"%{q.lower()}%"
+ params.extend([like_value] * 8)
+
+ return conditions, params
+
+ def list_files(
+ self,
+ status: str | None = None,
+ template: str | None = None,
+ period_format: str | None = None,
+ q: str | None = None,
+ current_key: str | None = None,
+ emitting_company: str | None = None,
+ document_type: str | None = None,
+ period: str | None = None,
+ confidence: str | None = None,
+ updated_at: str | None = None,
+ sort_by: str = "updated_at",
+ sort_order: str = "desc",
+ page: int = 1,
+ page_size: int = 25,
+ ) -> list[FileRecord]:
+ conditions, params = self._file_conditions(
+ status=status,
+ template=template,
+ period_format=period_format,
+ q=q,
+ current_key=current_key,
+ emitting_company=emitting_company,
+ document_type=document_type,
+ period=period,
+ confidence=confidence,
+ updated_at=updated_at,
+ )
+
+ query = "SELECT * FROM files"
+ if conditions:
+ query += " WHERE " + " AND ".join(conditions)
+ sortable_columns = {
+ "status": "status",
+ "current_key": "current_key",
+ "template_name": "template_name",
+ "period_format": "period_format",
+ "emitting_company": "emitting_company",
+ "document_type": "document_type",
+ "period": "period",
+ "confidence": "confidence",
+ "updated_at": "updated_at",
+ }
+ order_column = sortable_columns.get(sort_by, "updated_at")
+ order_direction = "ASC" if sort_order.lower() == "asc" else "DESC"
+ query += f" ORDER BY {order_column} {order_direction} LIMIT ? OFFSET ?"
+ params.extend([page_size, (page - 1) * page_size])
+
+ with self._connect() as conn:
+ rows = conn.execute(query, params).fetchall()
+
+ return [FileRecord.from_row(tuple(row)) for row in rows]
+
+ def count_files(
+ self,
+ status: str | None = None,
+ template: str | None = None,
+ period_format: str | None = None,
+ q: str | None = None,
+ current_key: str | None = None,
+ emitting_company: str | None = None,
+ document_type: str | None = None,
+ period: str | None = None,
+ confidence: str | None = None,
+ updated_at: str | None = None,
+ ) -> int:
+ conditions, params = self._file_conditions(
+ status=status,
+ template=template,
+ period_format=period_format,
+ q=q,
+ current_key=current_key,
+ emitting_company=emitting_company,
+ document_type=document_type,
+ period=period,
+ confidence=confidence,
+ updated_at=updated_at,
+ )
+ query = "SELECT COUNT(*) FROM files"
+ if conditions:
+ query += " WHERE " + " AND ".join(conditions)
+ with self._connect() as conn:
+ row = conn.execute(query, params).fetchone()
+ return int(row[0])
+
+ def get_status_counts(self) -> dict[str, int]:
+ with self._connect() as conn:
+ row = conn.execute(
+ """
+ SELECT
+ COUNT(*) AS total,
+ SUM(CASE WHEN status = 'processed' THEN 1 ELSE 0 END) AS processed,
+ SUM(CASE WHEN status = 'validated' THEN 1 ELSE 0 END) AS validated,
+ SUM(CASE WHEN status = 'not processed' THEN 1 ELSE 0 END) AS pending
+ FROM files
+ """
+ ).fetchone()
+ return {key: int(row[key] or 0) for key in ("total", "processed", "validated", "pending")}
+
+ def delete_all(self) -> None:
+ with self._connect() as conn:
+ conn.execute("DELETE FROM files")
+ conn.commit()
+
+ def rebuild_from_sidecars(self, storage: Any) -> list[FileRecord]:
+ """Rebuild the SQLite index from JSON sidecar metadata in storage."""
+ self.delete_all()
+ records: list[FileRecord] = []
+
+ for key in sorted(storage.list_objects(prefix="")):
+ if not key.endswith(".meta.json"):
+ continue
+ raw = storage.get_object(key)
+ if raw is None:
+ continue
+ if isinstance(raw, bytes):
+ raw = raw.decode("utf-8", errors="replace")
+ try:
+ metadata = json.loads(raw)
+ except json.JSONDecodeError:
+ continue
+ if not metadata:
+ continue
+ record = FileRecord.from_metadata(metadata)
+ self.upsert(record)
+ records.append(record)
+
+ logger.info(f"Rebuilt SQLite index from {len(records)} sidecar metadata files.")
+
+ return records
diff --git a/automatic-pdf-renamer/app/main.py b/automatic-pdf-renamer/app/main.py
new file mode 100644
index 0000000..88da323
--- /dev/null
+++ b/automatic-pdf-renamer/app/main.py
@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+
+from app.api.routes import router
+from app.config import load_config, load_templates
+from app.db import SQLiteFileRepository
+from app.storage import get_storage_backend
+
+
+def _build_storage(config) -> object:
+ backend_name = getattr(config, "storage_backend", "s3").lower()
+
+ if backend_name == "s3":
+ return get_storage_backend(
+ "s3",
+ endpoint_url=config.s3.endpoint_url,
+ aws_access_key_id=config.s3.access_key,
+ aws_secret_access_key=config.s3.secret_key,
+ bucket_name=config.s3.bucket_name,
+ region_name=config.s3.region,
+ )
+
+ storage_dir = getattr(config, "local_storage_dir", os.environ.get("LOCAL_STORAGE_DIR", "./storage"))
+ Path(storage_dir).mkdir(parents=True, exist_ok=True)
+ return get_storage_backend("file", base_dir=storage_dir)
+
+
+def create_app() -> FastAPI:
+ config = load_config()
+ registry = load_templates(os.environ.get("TEMPLATE_PATH", "./config.jsonc"))
+
+ app = FastAPI(
+ title="Automatic PDF Renamer",
+ version="0.1.0",
+ description="Offline API for PDF classification, validation, retraining, and consistency operations.",
+ )
+ app.state.config = config
+ app.state.template_registry = registry
+ app.state.storage = None
+ app.state.db = SQLiteFileRepository(db_path=config.sqlite_path)
+
+ app.add_middleware(
+ CORSMiddleware,
+ allow_origins=["*"],
+ allow_credentials=True,
+ allow_methods=["*"],
+ allow_headers=["*"],
+ )
+
+ @app.on_event("startup")
+ async def startup_event() -> None:
+ app.state.storage = _build_storage(config)
+ app.state.template_registry = load_templates(
+ os.environ.get("TEMPLATE_PATH", "./config.jsonc")
+ )
+ app.state.db = SQLiteFileRepository(db_path=config.sqlite_path)
+
+ app.include_router(router)
+
+ web_dir = Path("./web")
+ if web_dir.exists():
+ app.mount("/", StaticFiles(directory=str(web_dir), html=True), name="web")
+ app.mount("/static", StaticFiles(directory=str(web_dir)), name="web_static")
+
+ dist_dir = Path("./web/dist")
+ if dist_dir.exists():
+ app.mount("/static", StaticFiles(directory=str(dist_dir)), name="static")
+
+ return app
+
+
+app = create_app()
diff --git a/automatic-pdf-renamer/app/pipeline.py b/automatic-pdf-renamer/app/pipeline.py
index e132e1a..bd36c66 100644
--- a/automatic-pdf-renamer/app/pipeline.py
+++ b/automatic-pdf-renamer/app/pipeline.py
@@ -54,6 +54,7 @@ class ProcessingResult:
company: str = ""
document_type: str = ""
template_name: str = ""
+ period_format: str = ""
period: str = ""
status: str = SidecarStatus.NOT_PROCESSED.value
@@ -159,7 +160,7 @@ class DocumentProcessor:
# Extract resolved fields
template_name = resolved.get("template")
- period_format_name = resolved.get("period_format")
+ period_format = resolved.get("period_format")
additional_field_groups = resolved.get("additional_fields", [])
if not template_name:
@@ -171,6 +172,7 @@ class DocumentProcessor:
return result
result.template_name = template_name
+ result.period_format = period_format
# Step 4b: Extract additional fields from additional_fields lists
optional_fields = {}
@@ -186,7 +188,7 @@ class DocumentProcessor:
optional_fields[group_name] = field_value
# Step 5: Calculate period using period_format configuration
- if not period_format_name:
+ if not period_format:
result.errors.append(
f"No period_format resolved for company='{company}', "
f"document_type='{doc_type}'. Routing to review."
@@ -194,10 +196,10 @@ class DocumentProcessor:
result.status = SidecarStatus.NOT_PROCESSED.value
return result
- period_format_config = self.registry.get_period_format_config(period_format_name)
+ period_format_config = self.registry.get_period_format_config(period_format)
if period_format_config is None:
result.errors.append(
- f"Period format '{period_format_name}' not found in configuration"
+ f"Period format '{period_format}' not found in configuration"
)
result.status = SidecarStatus.NOT_PROCESSED.value
return result
@@ -224,6 +226,7 @@ class DocumentProcessor:
emission_date=emission_date,
period=period,
template_name=template_name,
+ period_format=period_format,
emitting_company=company,
original_filename=original_key.split("/")[-1],
**optional_fields,
@@ -244,6 +247,7 @@ class DocumentProcessor:
current_key=final_key,
status=SidecarStatus.PROCESSED,
template_name=template_name,
+ period_format=period_format,
emission_date=emission_date,
period=period,
emitting_company=company,
@@ -263,6 +267,7 @@ class DocumentProcessor:
"emission_date": emission_date,
"period": period,
"confidence": confidence,
+ "period_format": period_format,
"optional_fields": optional_fields,
},
)
diff --git a/automatic-pdf-renamer/app/sidecar.py b/automatic-pdf-renamer/app/sidecar.py
index 449ba76..3d96305 100644
--- a/automatic-pdf-renamer/app/sidecar.py
+++ b/automatic-pdf-renamer/app/sidecar.py
@@ -65,6 +65,7 @@ class Sidecar:
current_key: str = ""
status: str = SidecarStatus.NOT_PROCESSED.value
template_name: str = ""
+ period_format: str = ""
confidence: float = 0.0
emission_date: str = ""
period: str = ""
@@ -105,6 +106,7 @@ class Sidecar:
current_key=data.get("current_key", ""),
status=data.get("status", SidecarStatus.NOT_PROCESSED.value),
template_name=data.get("template_name", ""),
+ period_format=data.get("period_format", ""),
confidence=data.get("confidence", 0.0),
emission_date=data.get("emission_date", ""),
period=data.get("period", ""),
@@ -137,6 +139,7 @@ class Sidecar:
# Required common fields per PRD section 4.1
_REQUIRED_COMMON_FIELDS = {
"template_name",
+ "period_format",
"emission_date",
"emitting_company",
"document_type",
@@ -150,7 +153,7 @@ _REQUIRED_SIDECAR_FIELDS = {
"original_key",
"current_key",
"status",
- *"template_name emission_date emitting_company document_type period".split(),
+ *"template_name period_format emission_date emitting_company document_type period".split(),
"created_at",
"updated_at",
"events",
@@ -236,6 +239,7 @@ def build_sidecar(
current_key: str,
status: SidecarStatus,
template_name: str,
+ period_format: str,
emission_date: str,
period: str,
emitting_company: str,
@@ -252,6 +256,7 @@ def build_sidecar(
current_key: Current S3 key of the file.
status: Current processing status.
template_name: Document family/template name.
+ period_format: Format of the period string (e.g., "YYYY-MM", "YYYY-MM-DD").
emission_date: Date the document was emitted (ISO format YYYY-MM-DD).
period: Human-readable period string.
emitting_company: Company that emitted the document.
@@ -269,6 +274,7 @@ def build_sidecar(
current_key=current_key,
status=status.value,
template_name=template_name,
+ period_format=period_format,
confidence=confidence,
emission_date=emission_date,
period=period,
@@ -317,8 +323,10 @@ def get_sidecar_key(pdf_key: str) -> str:
pdf_key: The S3 key of the PDF file.
Returns:
- The corresponding sidecar key (e.g., 'files/doc.pdf' -> 'files/doc.pdf.meta.json').
+ The corresponding sidecar key (e.g., 'files/doc.pdf' -> 'files/doc.meta.json').
"""
+ if pdf_key.endswith(".pdf"):
+ return f"{pdf_key[:-4]}.meta.json"
return f"{pdf_key}.meta.json"
@@ -345,7 +353,7 @@ def write_atomic(
- delete_object(key: str)
- object_exists(key: str) -> bool
sidecar: The Sidecar to write.
- target_key: The final destination key (e.g., 'files/doc.pdf.meta.json').
+ target_key: The final destination key (e.g., 'files/doc.meta.json').
temp_prefix: Prefix for temporary keys (default: 'tmp/').
Raises:
diff --git a/automatic-pdf-renamer/main.py b/automatic-pdf-renamer/main.py
new file mode 100644
index 0000000..7572bbb
--- /dev/null
+++ b/automatic-pdf-renamer/main.py
@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+import uvicorn
+
+from app.main import app
+
+
+if __name__ == "__main__":
+ uvicorn.run(app, host="0.0.0.0", port=8000)
diff --git a/automatic-pdf-renamer/scripts/import_paperless.py b/automatic-pdf-renamer/scripts/import_paperless.py
index b8b8855..38db0c0 100755
--- a/automatic-pdf-renamer/scripts/import_paperless.py
+++ b/automatic-pdf-renamer/scripts/import_paperless.py
@@ -62,6 +62,9 @@ logging.basicConfig(
)
logger = logging.getLogger(__name__)
+PAPERLESS_URL = os.environ.get("PAPERLESS_URL", "")
+PAPERLESS_TOKEN = os.environ.get("PAPERLESS_TOKEN", "")
+
@dataclass
class PaperlessDocument:
@@ -400,6 +403,29 @@ class DocumentProcessor:
self.storage = storage
self.pdf_processor = PDFProcessor()
+ def _unique_target_key(self, target_key: str) -> str:
+ """Add an incremental suffix when a generated PDF key already exists."""
+ pdf_key = f"{target_key}.pdf"
+ if not self.storage.object_exists(pdf_key) and not self.storage.object_exists(
+ get_sidecar_key(pdf_key)
+ ):
+ return target_key
+
+ directory, separator, filename = target_key.rpartition("/")
+ extension = ".pdf" if filename.lower().endswith(".pdf") else ""
+ stem = filename[:-len(extension)] if extension else filename
+ counter = 2
+
+ while True:
+ candidate_filename = f"{stem} ({counter}){extension}"
+ candidate = f"{directory}{separator}{candidate_filename}"
+ candidate_pdf_key = f"{candidate}.pdf"
+ if not self.storage.object_exists(candidate_pdf_key) and not self.storage.object_exists(
+ get_sidecar_key(candidate_pdf_key)
+ ):
+ return candidate
+ counter += 1
+
def resolve_mapping(self, company: str, document_type: str) -> ResolvedMapping:
"""Resolve the mapping for a document."""
result = ResolvedMapping()
@@ -480,7 +506,7 @@ class DocumentProcessor:
Returns:
Tuple of (target_key, sidecar, errors)
- - target_key: Either files/{final_key} or pending/{paperless_id}
+ - target_key: Either files/{final_key}.pdf or pending/{paperless_id}.pdf
- sidecar: Sidecar metadata (or None if failed)
- errors: List of error messages
"""
@@ -593,7 +619,7 @@ class DocumentProcessor:
return self._route_to_pending(doc, errors)
# Step 7: Build sidecar for files/
- pdf_key = f"files/{final_key}"
+ pdf_key = self._unique_target_key(f"files/{final_key}.pdf")
sha256 = "" # Will be computed when writing
sidecar = build_sidecar(
@@ -602,6 +628,7 @@ class DocumentProcessor:
current_key=pdf_key,
status=SidecarStatus.PROCESSED,
template_name=template_name,
+ period_format=resolved.period_format,
emission_date=doc.created_date,
period=period,
emitting_company=company,
@@ -629,12 +656,9 @@ class DocumentProcessor:
errors: list[str]
) -> tuple[str, Sidecar | None, list[str]]:
"""Route document to pending/ with error sidecar."""
- pdf_key = f"pending/{doc.id}"
+ pdf_key = f"pending/{doc.id}.pdf"
- # Build sidecar with errors in optional_fields
- # We use optional_fields to store errors for pending documents
optional_fields = {"tags": doc.tags} if doc.tags else {}
- optional_fields["_import_errors"] = errors
# Build sidecar
sidecar = build_sidecar(
@@ -643,6 +667,7 @@ class DocumentProcessor:
current_key=pdf_key,
status=SidecarStatus.NOT_PROCESSED,
template_name="",
+ period_format="",
emission_date=doc.created_date,
period="",
emitting_company=doc.correspondent_name,
@@ -792,7 +817,7 @@ class Importer:
return True
# Write PDF
- self.storage.put_object(target_key + ".pdf", pdf_bytes)
+ self.storage.put_object(target_key, pdf_bytes)
logger.debug(f"Uploaded PDF to {target_key}")
# Update sidecar
diff --git a/automatic-pdf-renamer/user-story-0-paperless-ngx-initial-import.md b/automatic-pdf-renamer/user-stories/user-story-0-paperless-ngx-initial-import.md
similarity index 100%
rename from automatic-pdf-renamer/user-story-0-paperless-ngx-initial-import.md
rename to automatic-pdf-renamer/user-stories/user-story-0-paperless-ngx-initial-import.md
diff --git a/automatic-pdf-renamer/web/app.js b/automatic-pdf-renamer/web/app.js
new file mode 100644
index 0000000..f734406
--- /dev/null
+++ b/automatic-pdf-renamer/web/app.js
@@ -0,0 +1,236 @@
+const searchFilter = document.getElementById('search-filter');
+const body = document.getElementById('documents-body');
+const summaryCards = document.getElementById('summary-cards');
+const refreshButton = document.getElementById('refresh-btn');
+const rebuildButton = document.getElementById('rebuild-btn');
+const rebuildStatus = document.getElementById('rebuild-status');
+const listStatus = document.getElementById('list-status');
+const loadMoreButton = document.getElementById('load-more-btn');
+const loadSentinel = document.getElementById('load-sentinel');
+const columnFilters = [...document.querySelectorAll('[data-filter]')];
+const sortButtons = [...document.querySelectorAll('[data-sort]')];
+let sortBy = 'updated_at';
+let sortOrder = 'desc';
+let page = 1;
+let total = 0;
+let hasMore = true;
+let loading = false;
+let reloadAfterRequest = false;
+let loadedItems = [];
+
+const formatUpdated = (value) => {
+ if (!value) return '—';
+ try {
+ return new Date(value).toLocaleString('fr-FR');
+ } catch {
+ return value;
+ }
+};
+
+const escapeHtml = (value) => String(value ?? '').replace(/[&<>"']/g, (character) => ({
+ '&': '&',
+ '<': '<',
+ '>': '>',
+ '"': '"',
+ "'": ''',
+}[character]));
+
+const getListParams = () => {
+ const params = new URLSearchParams();
+ columnFilters.forEach((filter) => {
+ const value = filter.value.trim();
+ if (value) params.set(filter.dataset.filter, value);
+ });
+ const search = searchFilter.value.trim();
+ if (search) params.set('q', search);
+ params.set('sort_by', sortBy);
+ params.set('sort_order', sortOrder);
+ return params;
+};
+
+const badgeClass = (status) => {
+ const normalized = (status || 'unknown').trim().toLowerCase();
+ if (normalized === 'processed' || normalized === 'validated') return normalized;
+ if (normalized === 'not processed') return 'not processed';
+ return 'unknown';
+};
+
+const renderSummary = (totals) => {
+ summaryCards.innerHTML = `
+
Document detail
+Dashboard
+| + | + | + | + | + | + | + | + |
|---|---|---|---|---|---|---|---|
| + | + | + | + | + | + | + | + |
| Loading documents... | +|||||||