multiarch-docker-images/automatic-pdf-renamer/app/api/routes.py

from __future__ import annotations

import json
import logging
import os
import re
import threading
import uuid
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

from fastapi import APIRouter, Depends, HTTPException, Request, Response, status

from app.config import TemplateRegistry, load_templates
from app.db import FileRecord, SQLiteFileRepository
from app.period import format_period_from_emission_date
from app.pipeline import DocumentProcessor
from app.storage import StorageBackend
from app.model_manager import list_models, train_all_models
from app.ml import LocalMLPipeline

from .deps import get_current_user
from .models import (
    FileDetail,
    FileListItem,
    ConfigUpdateRequest,
    DocumentEditRequest,
    HealthResponse,
    ValidationRequest,
    ValidationResponse,
)

router = APIRouter(tags=["files"])

logger = logging.getLogger('uvicorn.error')


def _storage(request: Request) -> StorageBackend:
    storage = getattr(request.app.state, "storage", None)
    if storage is None:
        raise HTTPException(status_code=500, detail="Storage backend is not configured")
    return storage


def _db(request: Request) -> SQLiteFileRepository:
    db = getattr(request.app.state, "db", None)
    if db is None:
        raise HTTPException(status_code=500, detail="SQLite repository is not configured")
    return db


def _registry(request: Request):
    registry = getattr(request.app.state, "template_registry", None)
    if registry is None:
        registry = load_templates()
        request.app.state.template_registry = registry
    return registry


def _template_config_path() -> Path:
    configured_path = Path(os.environ.get("TEMPLATE_PATH", "./config.jsonc"))
    if configured_path.suffix.lower() not in {".json", ".jsonc"}:
        raise HTTPException(status_code=400, detail="The configured template file is not JSON or JSONC.")
    if not configured_path.is_file():
        raise HTTPException(status_code=404, detail="The configured template file does not exist.")
    return configured_path.resolve()


def _parse_file_metadata(raw: str | bytes | None) -> dict[str, Any]:
    if raw is None:
        return {}
    if isinstance(raw, bytes):
        raw = raw.decode("utf-8", errors="replace")
    try:
        return json.loads(raw)
    except json.JSONDecodeError:
        return {}


def _as_file_item(file_id: str, metadata: dict[str, Any]) -> FileListItem:
    return FileListItem(
        id=file_id,
        status=str(metadata.get("status", "unknown")),
        current_key=str(metadata.get("current_key", metadata.get("key", ""))),
        original_key=metadata.get("original_key"),
        template_name=metadata.get("template_name"),
        period_format=metadata.get("period_format"),
        document_type=metadata.get("document_type"),
        emitting_company=metadata.get("emitting_company"),
        period=metadata.get("period"),
        emission_date=metadata.get("emission_date"),
        confidence=metadata.get("confidence"),
        updated_at=metadata.get("updated_at"),
    )


@router.get("/health", response_model=HealthResponse)
async def health() -> HealthResponse:
    """Service health checkpoint."""
    logger.debug("API GET /health: health check requested")
    result = HealthResponse(status="ok", app="automatic-pdf-renamer")
    logger.debug("API GET /health: status=%s", result.status)
    return result


@router.get("/files", response_model=list[FileListItem])
async def list_files(
    request: Request,
    response: Response,
    current_user: dict[str, str] | None = Depends(get_current_user),
    status: str | None = None,
    template: str | None = None,
    period_format: str | None = None,
    q: str | None = None,
    current_key: str | None = None,
    emitting_company: str | None = None,
    document_type: str | None = None,
    period: str | None = None,
    confidence: str | None = None,
    updated_at: str | None = None,
    sort_by: str = "updated_at",
    sort_order: str = "desc",
    page: int = 1,
) -> list[FileListItem]:
    """List documents and report filtered and global counts in response headers."""
    logger.debug("API GET /files: user=%s page=%s sort_by=%s sort_order=%s", (current_user or {}).get("username", "anonymous"), page, sort_by, sort_order)
    logger.debug("API GET /files: active filters=%s", {key: value for key, value in {"status": status, "template": template, "period_format": period_format, "q": q, "current_key": current_key, "emitting_company": emitting_company, "document_type": document_type, "period": period, "confidence": confidence, "updated_at": updated_at}.items() if value is not None})
    storage = _storage(request)
    db = _db(request)
    filters = {
        "status": status,
        "template": template,
        "q": q,
        "current_key": current_key,
        "emitting_company": emitting_company,
        "document_type": document_type,
        "period": period,
        "period_format": period_format,
        "confidence": confidence,
        "updated_at": updated_at,
    }

    if db.count_files() == 0:
        logger.debug("SQLite is empty, rebuilding from sidecar metadata")
        for key in sorted(storage.list_objects(prefix="")):
            if not key.endswith(".meta.json"):
                continue
            metadata = _parse_file_metadata(storage.get_object(key))
            if metadata:
                db.upsert(FileRecord.from_metadata(metadata))

    records = db.list_files(
        **filters,
        sort_by=sort_by,
        sort_order=sort_order,
        page=page,
        page_size=25,
    )
    response.headers["X-Total-Count"] = str(db.count_files(**filters))
    response.headers["X-Status-Counts"] = json.dumps(db.get_status_counts())
    logger.debug("API GET /files: returning %s records; filtered_total=%s; status_counts=%s", len(records), response.headers["X-Total-Count"], response.headers["X-Status-Counts"])

    return [
        FileListItem(
            id=record.id,
            status=record.status,
            current_key=record.current_key,
            original_key=record.original_key,
            template_name=record.template_name,
            period_format=record.period_format,
            document_type=record.document_type,
            emitting_company=record.emitting_company,
            period=record.period,
            emission_date=record.emission_date,
            confidence=record.confidence,
            updated_at=record.updated_at,
        )
        for record in records
    ]


@router.get("/files/{file_id}", response_model=FileDetail)
async def get_file(
    request: Request,
    file_id: str,
    current_user: dict[str, str] | None = Depends(get_current_user),
) -> FileDetail:
    """Return metadata and extracted text for a single file."""
    logger.debug("API GET /files/%s: loading document metadata", file_id)
    storage = _storage(request)
    db = _db(request)
    record = db.get_by_id(file_id)

    if record is None:
        logger.debug(f"File '{file_id}' not found in SQLite, searching storage sidecars")
        keys = storage.list_objects(prefix="")
        for key in keys:
            if not key.endswith(".meta.json"):
                continue

            metadata = _parse_file_metadata(storage.get_object(key))
            if metadata and (metadata.get("sha256") == file_id or key == file_id):
                record = FileRecord.from_metadata(metadata)
                db.upsert(record)
                break

    if record is None:
        raise HTTPException(status_code=404, detail=f"File '{file_id}' not found")

    sidecar_metadata: dict[str, Any] = {}
    if record.current_key:
        metadata_key = f"{record.current_key.split('.')[0]}.meta.json" if record.current_key.endswith(".pdf") else f"{record.current_key}.meta.json"
        sidecar_metadata = _parse_file_metadata(
            storage.get_object(metadata_key)
        )
    if not sidecar_metadata or sidecar_metadata.get("sha256") not in {
        record.sha256,
        record.id,
    }:
        for key in storage.list_objects(prefix=""):
            if not key.endswith(".meta.json"):
                continue
            candidate = _parse_file_metadata(storage.get_object(key))
            if candidate.get("sha256") in {record.sha256, record.id}:
                sidecar_metadata = candidate
                break

    metadata = {
        "schema_version": sidecar_metadata.get("schema_version"),
        "sha256": record.sha256,
        "status": record.status,
        "current_key": record.current_key,
        "original_key": record.original_key,
        "template_name": record.template_name,
        "period_format": record.period_format,
        "document_type": record.document_type,
        "emitting_company": record.emitting_company,
        "period": record.period,
        "emission_date": record.emission_date,
        "confidence": record.confidence,
        "updated_at": record.updated_at,
        "created_at": record.created_at,
        "extracted_text": record.extracted_text,
        "optional_fields": record.optional_fields,
        "history": sidecar_metadata.get("events", []),
    }
    item = _as_file_item(record.id, metadata)
    result = FileDetail(
        id=item.id,
        status=item.status,
        current_key=item.current_key,
        original_key=item.original_key,
        template_name=item.template_name,
        period_format=item.period_format,
        document_type=item.document_type,
        emitting_company=item.emitting_company,
        period=item.period,
        emission_date=item.emission_date,
        confidence=item.confidence,
        updated_at=item.updated_at,
        schema_version=metadata["schema_version"],
        created_at=metadata["created_at"],
        extracted_text=record.extracted_text,
        optional_fields=record.optional_fields,
        history=metadata["history"] if isinstance(metadata["history"], list) else [],
    )
    logger.debug("API GET /files/%s: loaded status=%s current_key=%s optional_field_count=%s", file_id, result.status, result.current_key, len(result.optional_fields or {}))
    return result


@router.post("/files/{file_id}/suggestions")
async def get_document_model_suggestions(
    request: Request,
    file_id: str,
    payload: dict[str, Any],
    current_user: dict[str, str] | None = Depends(get_current_user),
) -> dict[str, Any]:
    """Return confidence-gated model suggestions without delaying document rendering."""
    db = _db(request)
    record = db.get_by_id(file_id)
    if record is None:
        raise HTTPException(status_code=404, detail=f"File '{file_id}' not found")

    registry = _registry(request)
    threshold = float(os.environ.get("CONFIDENCE_THRESHOLD", "0.8"))
    ml = LocalMLPipeline(confidence_threshold=threshold)
    text = record.extracted_text or ""
    company = str(payload.get("emitting_company") or record.emitting_company or "").strip()
    document_type = str(payload.get("document_type") or record.document_type or "").strip()
    current_fields = dict(record.optional_fields or {})
    submitted_fields = payload.get("optional_fields")
    if isinstance(submitted_fields, dict):
        current_fields.update({
            str(name): str(value)
            for name, value in submitted_fields.items()
            if value is not None and str(value).strip()
        })
    suggestions, field_decisions = ml.suggest_document_fields(
        text,
        company,
        document_type,
        current_fields,
        registry.get_additional_fields(),
    )
    field_decisions = {
        **field_decisions,
        "emission_date": "unavailable: no valid configured date or stored date",
        "period": "skipped: missing date or period format",
        "current_key": "skipped: missing template or key pattern",
    }

    # Recommend the first valid date matched in document order by configured regexes.
    # This is deterministic content extraction, not an ML prediction.
    try:
        recommended_date, date_error = DocumentProcessor(
            registry, ml_pipeline=ml
        )._extract_emission_date(text)
        if not recommended_date:
            # The stored date was originally produced by the same configured
            # content-extraction pipeline. Keep it available as a recommendation
            # when the current extraction configuration cannot reproduce it.
            stored_date = str(getattr(record, "emission_date", "") or "").strip()
            if stored_date:
                recommended_date = stored_date
                field_decisions["emission_date"] = "content: using stored date"
        if recommended_date:
            suggestions["emission_date"] = {
                "value": recommended_date,
                "source": "content",
            }
            field_decisions["emission_date"] = f"content: {recommended_date}"
        elif date_error:
            field_decisions["emission_date"] = f"unavailable: {date_error}"
    except Exception as exc:
        stored_date = str(getattr(record, "emission_date", "") or "").strip()
        if stored_date:
            suggestions["emission_date"] = {
                "value": stored_date,
                "source": "content",
            }
            field_decisions["emission_date"] = f"content: using stored date {stored_date}"
        else:
            field_decisions["emission_date"] = f"unavailable: extraction failed ({exc})"

    derived_template = str(payload.get("template_name") or record.template_name or "")
    derived_format = str(payload.get("period_format") or record.period_format or "")

    try:
        raw_date = str(payload.get("emission_date") or record.emission_date or "")
        emission_date = datetime.strptime(raw_date, "%Y-%m-%d").date()
        period_config = registry.get_period_format_config(derived_format)
        if period_config is not None:
            calculated_period = format_period_from_emission_date(emission_date, period_config.to_dict())
            suggestions["period"] = {
                "value": calculated_period,
                "source": "derived",
            }
            field_decisions["period"] = f"derived: {calculated_period}"
            key_pattern = registry.get_key_pattern(derived_template)
            if key_pattern:
                optional_fields = payload.get("optional_fields")
                if not isinstance(optional_fields, dict):
                    optional_fields = record.optional_fields or {}
                pattern_values = dict(optional_fields)
                pattern_values.update({
                    "company": company,
                    "document_type": document_type,
                    "emission_date": emission_date.isoformat(),
                    "period": calculated_period,
                    "period_format": derived_format,
                    "template_name": derived_template,
                    "emitting_company": company,
                    "additional_fields": ", ".join(str(value) for value in optional_fields.values()),
                    "original_filename": (record.original_key or "").rsplit("/", 1)[-1],
                })
                for _ in range(20):
                    try:
                        calculated_key = key_pattern.format(**pattern_values).strip("/")
                        break
                    except KeyError as error:
                        pattern_values[str(error.args[0])] = "unknown"
                else:
                    calculated_key = ""
                if calculated_key:
                    if not calculated_key.startswith("files/"):
                        calculated_key = f"files/{calculated_key}"
                    if not calculated_key.lower().endswith(".pdf"):
                        calculated_key = f"{calculated_key}.pdf"
                    suggestions["current_key"] = {
                        "value": calculated_key,
                        "source": "derived",
                    }
                    field_decisions["current_key"] = f"derived: {calculated_key}"
                else:
                    field_decisions["current_key"] = "unavailable: key pattern could not be formatted"
            else:
                field_decisions["current_key"] = "skipped: template has no key pattern"
        else:
            field_decisions["period"] = f"unavailable: unknown period format {derived_format!r}"
    except (TypeError, ValueError) as exc:
        field_decisions["period"] = f"unavailable: {exc}"

    logger.debug(
        "API POST /files/%s/suggestions decisions=%s",
        file_id,
        field_decisions,
    )
    return {"suggestions": suggestions, "confidence_threshold": threshold}


@router.get("/files/{file_id}/pdf")
async def get_file_pdf(
    request: Request,
    file_id: str,
    current_user: dict[str, str] | None = Depends(get_current_user),
) -> Response:
    """Stream a PDF by key from storage when present."""
    logger.debug("API GET /files/%s/pdf: resolving PDF storage key", file_id)
    storage = _storage(request)
    db = _db(request)
    record = db.get_by_id(file_id)

    key = record.current_key if record else None
    if key and storage.object_exists(key):
        payload = storage.get_object(key)
        if payload is not None:
            logger.debug(f"Streaming PDF for file_id={file_id} from key={key}")
            return Response(content=payload, media_type="application/pdf")
        raise HTTPException(status_code=404, detail="PDF bytes not found for the file")

    logger.debug(f"PDF for file_id={file_id} not found in SQLite, searching storage sidecars")
    keys = storage.list_objects(prefix="")
    for key in keys:
        if key.endswith(".meta.json"):
            metadata = _parse_file_metadata(storage.get_object(key))
            if metadata and (metadata.get("sha256") == file_id or key == file_id):
                current_key = metadata.get("current_key") or key.replace(".meta.json", "")
                pdf_keys = [current_key]
                if not current_key.lower().endswith(".pdf"):
                    pdf_keys.append(f"{current_key}.pdf")
                for pdf_key in pdf_keys:
                    payload = storage.get_object(pdf_key)
                    if payload is not None:
                        return Response(content=payload, media_type="application/pdf")
                raise HTTPException(status_code=404, detail="PDF bytes not found for the file")

    raise HTTPException(status_code=404, detail=f"PDF for '{file_id}' not found")


@router.post("/files/{file_id}/edit", response_model=FileDetail)
async def edit_file(
    request: Request,
    file_id: str,
    payload: DocumentEditRequest,
    current_user: dict[str, str] | None = Depends(get_current_user),
) -> FileDetail:
    """Recalculate and persist user-edited document metadata and its storage key."""
    logger.debug("API POST /files/%s/edit: template=%s period_format=%s company=%s document_type=%s optional_fields=%s", file_id, payload.template_name, payload.period_format, payload.emitting_company, payload.document_type, sorted((payload.optional_fields or {}).keys()))
    storage = _storage(request)
    db = _db(request)
    registry = _registry(request)
    record = db.get_by_id(file_id)
    if record is None:
        raise HTTPException(status_code=404, detail=f"File '{file_id}' not found")

    template_name = payload.template_name.strip()
    period_format = payload.period_format.strip()
    emitting_company = payload.emitting_company.strip()
    document_type = payload.document_type.strip()
    if not emitting_company or not document_type:
        raise HTTPException(status_code=422, detail="Company and document type are required")

    if registry.get_template(template_name) is None:
        raise HTTPException(status_code=422, detail=f"Unknown template '{template_name}'")
    period_config = registry.get_period_format_config(period_format)
    if period_config is None:
        raise HTTPException(status_code=422, detail=f"Unknown period format '{period_format}'")

    emission_date = payload.emission_date.isoformat()
    try:
        period = format_period_from_emission_date(
            payload.emission_date,
            period_config.to_dict(),
        )
    except (TypeError, ValueError) as error:
        raise HTTPException(status_code=422, detail=f"Unable to calculate period: {error}") from error

    updated_optional_fields = {
        str(label).strip(): value
        for label, value in (payload.optional_fields or {}).items()
        if str(label).strip()
    }

    key_pattern = registry.get_key_pattern(template_name)
    if not key_pattern:
        raise HTTPException(status_code=422, detail=f"Template '{template_name}' has no key pattern")
    pattern_values = dict(updated_optional_fields)
    pattern_values.update({
        "company": emitting_company,
        "document_type": document_type,
        "emission_date": emission_date,
        "period": period,
        "period_format": period_format,
        "template_name": template_name,
        "emitting_company": emitting_company,
        "additional_fields": ", ".join(updated_optional_fields.values()),
        "original_filename": record.original_key.rsplit("/", 1)[-1],
    })
    for _ in range(20):
        try:
            formatted_key = key_pattern.format(**pattern_values).strip("/")
            break
        except KeyError as error:
            pattern_values[str(error.args[0])] = "unknown"
    else:
        raise HTTPException(status_code=422, detail="Template key pattern has too many unresolved fields")

    if not formatted_key or any(part in {".", ".."} for part in formatted_key.split("/")):
        raise HTTPException(status_code=422, detail="Template produced an invalid document key")
    new_current_key = formatted_key if formatted_key.startswith("files/") else f"files/{formatted_key}"
    new_pdf_key = new_current_key if new_current_key.lower().endswith(".pdf") else f"{new_current_key}.pdf"
    new_sidecar_key = f"{new_current_key}.meta.json"

    source_sidecar_key = ""
    sidecar_metadata: dict[str, Any] = {}
    known_sidecar_key = f"{record.current_key.split('.')[0]}.meta.json" if record.current_key.endswith(".pdf") else f"{record.current_key}.meta.json"
    if known_sidecar_key:
        candidate = _parse_file_metadata(storage.get_object(known_sidecar_key))
        if candidate and candidate.get("sha256") in {record.sha256, record.id}:
            source_sidecar_key = known_sidecar_key
            sidecar_metadata = candidate
    if not source_sidecar_key:
        for key in storage.list_objects(prefix=""):
            if not key.endswith(".meta.json"):
                continue
            candidate = _parse_file_metadata(storage.get_object(key))
            if candidate and (
                candidate.get("sha256") in {record.sha256, record.id}
                or candidate.get("current_key") == record.current_key
            ):
                source_sidecar_key = key
                sidecar_metadata = candidate
                break
    if not source_sidecar_key:
        raise HTTPException(status_code=404, detail="Document sidecar was not found")

    source_current_key = str(sidecar_metadata.get("current_key") or record.current_key)
    source_pdf_key = ""
    pdf_candidates = [source_current_key]
    if source_current_key and not source_current_key.lower().endswith(".pdf"):
        pdf_candidates.append(f"{source_current_key}.pdf")
    for candidate in pdf_candidates:
        if candidate and storage.object_exists(candidate):
            source_pdf_key = candidate
            break

    if storage.object_exists(new_pdf_key) and new_pdf_key != source_pdf_key:
        raise HTTPException(status_code=409, detail="The recalculated PDF key is already in use")
    destination_sidecar = _parse_file_metadata(storage.get_object(new_sidecar_key))
    if (
        destination_sidecar
        and new_sidecar_key != source_sidecar_key
        and destination_sidecar.get("sha256") not in {record.sha256, record.id}
    ):
        raise HTTPException(status_code=409, detail="The recalculated sidecar key is already in use")

    updated_at = datetime.now(timezone.utc).isoformat()
    old_values = {}
    new_values = {}
    if template_name != record.template_name:
        old_values["template_name"] = record.template_name
        new_values["template_name"] = template_name
    if period_format != record.period_format:
        old_values["period_format"] = record.period_format
        new_values["period_format"] = period_format
    if emitting_company != record.emitting_company:
        old_values["emitting_company"] = record.emitting_company
        new_values["emitting_company"] = emitting_company
    if document_type != record.document_type:
        old_values["document_type"] = record.document_type
        new_values["document_type"] = document_type
    if emission_date != record.emission_date:
        old_values["emission_date"] = record.emission_date
        new_values["emission_date"] = emission_date
    if period != record.period:
        old_values["period"] = record.period
        new_values["period"] = period
    if new_pdf_key != record.current_key:
        old_values["current_key"] = record.current_key
        new_values["current_key"] = new_pdf_key
    if updated_optional_fields != record.optional_fields:
        old_values["optional_fields"] = dict(record.optional_fields)
        new_values["optional_fields"] = dict(updated_optional_fields)
    sidecar_metadata.update(new_values)
    sidecar_metadata["optional_fields"] = updated_optional_fields
    sidecar_metadata["status"] = "validated"
    sidecar_metadata["updated_at"] = updated_at
    events = sidecar_metadata.get("events")
    if not isinstance(events, list):
        events = []
        sidecar_metadata["events"] = events
    events.append({
        "type": "corrected",
        "timestamp": updated_at,
        "payload": {"before": old_values, "after": new_values},
    })

    copied_pdf = bool(source_pdf_key and source_pdf_key != new_pdf_key)
    if copied_pdf:
        storage.copy_object(source_pdf_key, new_pdf_key)
    try:
        storage.put_object(new_sidecar_key, json.dumps(sidecar_metadata, ensure_ascii=False, indent=2))
    except Exception:
        if copied_pdf:
            storage.delete_object(new_pdf_key)
        raise

    record.template_name = template_name
    record.period_format = period_format
    record.emitting_company = emitting_company
    record.document_type = document_type
    record.emission_date = emission_date
    record.period = period
    record.current_key = new_pdf_key
    record.optional_fields = updated_optional_fields
    record.status = "validated"
    record.updated_at = updated_at
    db.upsert(record)

    if source_sidecar_key != new_sidecar_key:
        storage.delete_object(source_sidecar_key)
    if source_pdf_key and source_pdf_key != new_pdf_key:
        storage.delete_object(source_pdf_key)

    return await get_file(request, record.id, current_user=current_user)


@router.post("/files/{file_id}/validate", response_model=ValidationResponse)
async def validate_file(
    request: Request,
    file_id: str,
    payload: ValidationRequest,
    current_user: dict[str, str] | None = Depends(get_current_user),
) -> ValidationResponse:
    """Apply a manual validation update and persist it back to the sidecar metadata."""
    storage = _storage(request)
    db = _db(request)
    record = db.get_by_id(file_id)

    if record is None:
        logger.debug(f"File '{file_id}' not found in SQLite, searching storage sidecars")
        keys = storage.list_objects(prefix="")
        for key in keys:
            if key.endswith(".meta.json"):
                metadata = _parse_file_metadata(storage.get_object(key))
                if metadata and (metadata.get("sha256") == file_id or key == file_id):
                    record = FileRecord.from_metadata(metadata)
                    db.upsert(record)
                    break

    if record is None:
        raise HTTPException(status_code=404, detail=f"File '{file_id}' not found")

    metadata = dict(record.optional_fields)
    if payload.template_id:
        record.template_name = payload.template_id
    if payload.fields:
        record.optional_fields = {**record.optional_fields, **payload.fields}
        metadata = record.optional_fields

    record.status = "validated"
    record.updated_at = "now"
    db.upsert(record)

    sidecar_key = f"{record.current_key}.meta.json" if record.current_key else f"{file_id}.meta.json"
    storage.put_object(sidecar_key, json.dumps({
        "sha256": record.sha256,
        "original_key": record.original_key,
        "current_key": record.current_key,
        "status": record.status,
        "template_name": record.template_name,
        "period_format": record.period_format,
        "document_type": record.document_type,
        "emitting_company": record.emitting_company,
        "period": record.period,
        "emission_date": record.emission_date,
        "confidence": record.confidence,
        "optional_fields": record.optional_fields,
        "extracted_text": record.extracted_text,
        "updated_at": record.updated_at,
    }, ensure_ascii=False, indent=2))

    return ValidationResponse(
        id=str(record.sha256 or file_id),
        status="validated",
        current_key=record.current_key,
        template_name=record.template_name,
        fields=record.optional_fields,
        updated=True,
    )


@router.get("/document-options")
async def document_edit_options(
    request: Request,
    current_user: dict[str, str] | None = Depends(get_current_user),
) -> dict[str, Any]:
    """Return configured templates and existing document values for edit suggestions."""
    logger.debug("API GET /document-options: loading template and field options")
    registry = _registry(request)
    db = _db(request)
    records = db.list_files(page_size=10000)

    companies = {
        str(company).strip()
        for company in registry.get_companies()
        if str(company).strip()
    } | {
        str(record.emitting_company).strip()
        for record in records
        if record.emitting_company and str(record.emitting_company).strip()
    }
    document_types = {
        str(document_type).strip()
        for document_type in registry.get_document_types()
        if str(document_type).strip()
    } | {
        str(record.document_type).strip()
        for record in records
        if record.document_type and str(record.document_type).strip()
    }
    optional_field_values_by_name: dict[str, set[str]] = {}
    for name, values in registry.get_additional_fields().items():
        field_name = str(name).strip()
        if field_name:
            optional_field_values_by_name[field_name] = {
                str(value).strip()
                for value in values
                if value is not None and str(value).strip()
            }
    optional_field_names = set(optional_field_values_by_name)
    for record in records:
        for name, value in (record.optional_fields or {}).items():
            field_name = str(name).strip()
            field_value = str(value).strip() if value is not None else ""
            if field_name:
                optional_field_names.add(field_name)
                if field_value:
                    optional_field_values_by_name.setdefault(field_name, set()).add(field_value)
    optional_field_values = {
        name: sorted(values)
        for name, values in sorted(optional_field_values_by_name.items())
    }

    return {
        "templates": registry.list_all_templates(),
        "period_formats": registry.list_period_formats(),
        "companies": sorted(companies),
        "document_types": sorted(document_types),
        "optional_field_names": sorted(optional_field_names),
        "optional_field_values": optional_field_values,
    }


@router.get("/config")
async def get_config(
    current_user: dict[str, str] | None = Depends(get_current_user),
) -> dict[str, str]:
    """Return the configured JSONC file for editing in the dashboard."""
    path = _template_config_path()
    try:
        return {"content": path.read_text(encoding="utf-8")}
    except OSError as error:
        raise HTTPException(status_code=500, detail="Unable to read the configured template file.") from error


@router.put("/config")
async def update_config(
    payload: ConfigUpdateRequest,
    request: Request,
    current_user: dict[str, str] | None = Depends(get_current_user),
) -> dict[str, bool]:
    """Validate and save JSONC configuration, then activate it without restarting the process."""
    registry = TemplateRegistry()
    try:
        registry.load_from_content(payload.content)
    except (json.JSONDecodeError, ValueError, TypeError, AttributeError, KeyError, re.error) as error:
        raise HTTPException(status_code=422, detail=f"Invalid configuration: {error}") from error

    path = _template_config_path()
    try:
        with path.open("w", encoding="utf-8") as config_file:
            config_file.write(payload.content)
            config_file.flush()
            os.fsync(config_file.fileno())
        request.app.state.template_registry = registry
    except OSError as error:
        raise HTTPException(status_code=500, detail="Unable to save the configured template file.") from error

    return {"saved": True, "reloaded": True}


@router.get("/templates")
async def list_templates(
    request: Request,
    current_user: dict[str, str] | None = Depends(get_current_user),
) -> list[dict[str, Any]]:
    """List all templates configured for the service."""
    registry = _registry(request)
    items: list[dict[str, Any]] = []
    for template_name in registry.list_all_templates():
        template = registry.get_template(template_name)
        if template is None:
            continue
        items.append({
            "name": template.name,
            "key_pattern": template.key_pattern,
        })
    return items


@router.get("/models")
async def get_models(
    current_user: dict[str, str] | None = Depends(get_current_user),
) -> dict[str, Any]:
    """Report whether each model in the current complete set is available."""
    result = list_models()
    logger.debug("API GET /models: complete=%s", result.get("complete"))
    return result


@router.post("/models/train", status_code=202)
async def train_models(
    request: Request,
    current_user: dict[str, str] | None = Depends(get_current_user),
) -> dict[str, Any]:
    """Start one background job that replaces the complete model set."""
    jobs = getattr(request.app.state, "model_training_jobs", None)
    if jobs is None:
        jobs = {}
        request.app.state.model_training_jobs = jobs
    if any(job.get("status") == "running" for job in jobs.values()):
        raise HTTPException(status_code=409, detail="A full model retraining job is already running.")

    records = _db(request).list_files(status="validated", page=1, page_size=100000)
    job_id = uuid.uuid4().hex
    jobs[job_id] = {
        "job_id": job_id,
        "status": "running",
        "progress": 0,
        "stage": "Starting model training",
        "message": "Preparing validated documents",
    }

    def update_progress(progress: int, message: str) -> None:
        jobs[job_id].update({
            "progress": max(0, min(100, int(progress))),
            "stage": message,
            "message": message,
        })

    def run_training() -> None:
        try:
            result = train_all_models(records, progress_callback=update_progress)
            jobs[job_id].update({
                "status": "completed",
                "progress": 100,
                "stage": "Completed",
                "message": "All models have been retrained and replaced.",
                "result": result,
            })
        except Exception as error:
            logger.exception("Full model retraining job %s failed", job_id)
            jobs[job_id].update({
                "status": "failed",
                "stage": "Failed",
                "message": str(error),
            })

    threading.Thread(target=run_training, name=f"model-training-{job_id[:8]}", daemon=True).start()
    logger.debug("API POST /models/train: started full retraining job %s", job_id)
    return {"job_id": job_id, "status": "running"}


@router.get("/jobs/{job_id}")
async def get_job_status(
    request: Request,
    job_id: str,
    current_user: dict[str, str] | None = Depends(get_current_user),
) -> dict[str, Any]:
    """Return current progress for a background model-training job."""
    jobs = getattr(request.app.state, "model_training_jobs", {})
    job = jobs.get(job_id)
    if job is None:
        raise HTTPException(status_code=404, detail="Job not found.")
    return dict(job)


@router.get("/consistency")
async def list_consistency_flags(
    request: Request,
    current_user: dict[str, str] | None = Depends(get_current_user),
) -> list[dict[str, Any]]:
    """Return consistency anomalies. In the current implementation, this is a lightweight placeholder."""
    logger.debug("API GET /consistency: scanning storage and database consistency")
    storage = _storage(request)
    db = _db(request)
    anomalies: list[dict[str, Any]] = []
    for key in storage.list_objects(prefix=""):
        if key.endswith(".meta.json"):
            metadata = _parse_file_metadata(storage.get_object(key))
            if metadata and metadata.get("status") in {"processed", "validated"}:
                current_key = metadata.get("current_key", key.replace(".meta.json", ""))
                if not storage.object_exists(current_key):
                    anomalies.append({
                        "kind": "missing_object",
                        "key": key,
                        "message": "Sidecar points to an object that is no longer present.",
                    })
    for record in db.list_files():
        if record.status in {"processed", "validated"} and not storage.object_exists(record.current_key):
            anomalies.append({
                "kind": "missing_object",
                "key": record.current_key,
                "message": "SQLite record points to a missing object.",
            })
    return anomalies


@router.post("/consistency/rebuild")
async def rebuild_consistency(
    request: Request,
    current_user: dict[str, str] | None = Depends(get_current_user),
) -> dict[str, Any]:
    """Rebuild SQLite from sidecar metadata and return the completed result."""
    storage = _storage(request)
    db = _db(request)
    rebuilt = db.rebuild_from_sidecars(storage)
    request.app.state.last_rebuild = {
        "status": "completed",
        "message": "SQLite rebuilt from sidecar metadata.",
        "count": len(rebuilt),
    }
    return request.app.state.last_rebuild