Added end-to-end tests

automatic-pdf-renamer/tests/e2e/README.md

@@ -0,0 +1,26 @@
+# Docker-based end-to-end tests
+
+The E2E suite starts the real API in a disposable Docker container using the existing `FileStorage` backend. It does not start the dashboard, connect to S3/Garage, or enable background workers.
+
+## Requirements
+
+- Docker Engine with the `docker` CLI
+- Python 3.11+
+- `pytest` (install with `python -m pip install -r tests/e2e/requirements.txt`)
+
+Run from `automatic-pdf-renamer/`:
+
+```sh
+python -m pip install -r tests/e2e/requirements.txt
+pytest -q tests/e2e
+```
+
+The suite builds the application image once per pytest session. Each test creates a baseline fixture tree (PDFs plus `.meta.json` sidecars) in its own temporary workspace, starts a new container with a new SQLite database, rebuilds SQLite from sidecars, and executes ordered HTTP actions. After each action, assertions can inspect the API, the host-mounted filesystem, and SQLite.
+
+Set `E2E_DOCKER_IMAGE` to override the temporary image tag. Set `E2E_STARTUP_TIMEOUT` to adjust the bounded startup wait. The suite uses a dynamically allocated localhost port and removes each container and temporary workspace when finished.
+
+Fixtures are checked-in immutable inputs under `tests/e2e/fixtures/baseline/`. The baseline includes an empty PDF and a valid one-page PDF, each with its matching `.meta.json` sidecar. Each test copies these files into its own temporary workspace; tests do not generate PDF or sidecar contents in Python.
+
+## Safety
+
+The suite always overrides storage and SQLite paths with paths inside its temporary workspace, and sets `APP_STORAGE_BACKEND=file`. It does not accept a production base URL or reuse a developer database. Do not modify the fixture source during a run.

automatic-pdf-renamer/tests/e2e/conftest.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import shutil
+import time
+import uuid
+from pathlib import Path
+from typing import Any, Iterator
+
+import pytest
+
+APP_DIR = Path(__file__).resolve().parents[2]
+
+def _run(command: list[str], *, timeout: int = 120) -> str:
+    result = subprocess.run(
+        command,
+        cwd=APP_DIR,
+        check=True,
+        capture_output=True,
+        text=True,
+        timeout=timeout,
+    )
+    return result.stdout.strip()
+
+
+def _write_baseline(workspace: Path) -> None:
+    """Copy checked-in PDF and sidecar fixtures into an isolated test workspace."""
+    storage = workspace / "storage"
+    fixture_dir = Path(__file__).resolve().parent / "fixtures" / "baseline"
+    if not fixture_dir.is_dir():
+        raise FileNotFoundError(f"E2E fixture directory does not exist: {fixture_dir}")
+    shutil.copytree(fixture_dir, storage)
+
+
+@pytest.fixture(scope="session")
+def docker_image() -> Iterator[str]:
+    image = os.environ.get("E2E_DOCKER_IMAGE", f"automatic-pdf-renamer-e2e:{uuid.uuid4().hex[:10]}")
+    _run(["docker", "build", "-t", image, "-f", "Dockerfile", "."], timeout=600)
+    try:
+        yield image
+    finally:
+        if not os.environ.get("E2E_DOCKER_IMAGE"):
+            subprocess.run(["docker", "image", "rm", "-f", image], capture_output=True, text=True)
+
+
+@pytest.fixture
+def e2e_service(tmp_path: Path, docker_image: str) -> Iterator[dict[str, Any]]:
+    workspace = tmp_path / "workspace"
+    _write_baseline(workspace)
+    db_path = workspace / "renamer.db"
+    name = f"pdf-renamer-e2e-{uuid.uuid4().hex[:12]}"
+    _run(
+        [
+            "docker", "run", "-d", "--rm", "--name", name,
+            "-p", "127.0.0.1::8000",
+            "-e", "APP_STORAGE_BACKEND=file",
+            "-e", "LOCAL_STORAGE_DIR=/e2e/storage",
+            "-e", "SQLITE_PATH=/e2e/renamer.db",
+            "-e", "DEBUG=false",
+            "-v", f"{workspace.resolve()}:/e2e",
+            docker_image,
+        ],
+        timeout=60,
+    )
+    try:
+        port = _run(["docker", "port", name, "8000/tcp"]).splitlines()[0].rsplit(":", 1)[1]
+        base_url = f"http://127.0.0.1:{port}"
+        deadline = time.monotonic() + float(os.environ.get("E2E_STARTUP_TIMEOUT", "45"))
+        import urllib.error
+        import urllib.request
+
+        while time.monotonic() < deadline:
+            try:
+                with urllib.request.urlopen(f"{base_url}/health", timeout=2) as response:
+                    if response.status == 200:
+                        break
+            except (OSError, urllib.error.URLError):
+                time.sleep(0.25)
+        else:
+            logs = subprocess.run(["docker", "logs", name], capture_output=True, text=True).stdout
+            raise RuntimeError(f"Application did not become healthy. Container logs:\n{logs}")
+
+        yield {"base_url": base_url, "workspace": workspace, "storage": workspace / "storage", "db_path": db_path}
+    finally:
+        subprocess.run(["docker", "stop", name], capture_output=True, text=True, timeout=30)
+
+
+def request_json(
+    base_url: str,
+    method: str,
+    path: str,
+    body: dict[str, Any] | None = None,
+) -> tuple[int, Any, dict[str, str]]:
+    import urllib.error
+    import urllib.request
+
+    payload = None if body is None else json.dumps(body).encode("utf-8")
+    request = urllib.request.Request(
+        base_url + path,
+        data=payload,
+        method=method,
+        headers={"Content-Type": "application/json"} if payload is not None else {},
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=10) as response:
+            raw = response.read()
+            try:
+                content: Any = json.loads(raw) if raw else None
+            except json.JSONDecodeError:
+                content = raw.decode("utf-8", errors="replace")
+            return response.status, content, dict(response.headers.items())
+    except urllib.error.HTTPError as error:
+        raw = error.read()
+        try:
+            content = json.loads(raw) if raw else None
+        except json.JSONDecodeError:
+            content = raw.decode("utf-8", errors="replace")
+        return error.code, content, dict(error.headers.items())

automatic-pdf-renamer/tests/e2e/fixtures/baseline/inbox/empty.meta.json

@@ -0,0 +1,19 @@
+{
+  "schema_version": "1.0",
+  "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "original_key": "inbox/empty.pdf",
+  "current_key": "inbox/empty.pdf",
+  "status": "not processed",
+  "template_name": "default",
+  "period_format": "monthly",
+  "confidence": 0,
+  "emission_date": "",
+  "period": "",
+  "emitting_company": "",
+  "document_type": "",
+  "optional_fields": {},
+  "extracted_text": "",
+  "created_at": "2026-01-01T00:00:00+00:00",
+  "updated_at": "2026-01-01T00:00:00+00:00",
+  "events": []
+}

automatic-pdf-renamer/tests/e2e/fixtures/baseline/pending/valid.meta.json

@@ -0,0 +1,19 @@
+{
+  "schema_version": "1.0",
+  "sha256": "b6c81b0f56ce2237b8d121f1066ed699fc32cd18a587e337cb7273d63e862fa4",
+  "original_key": "pending/valid.pdf",
+  "current_key": "pending/valid.pdf",
+  "status": "processed",
+  "template_name": "default",
+  "period_format": "monthly",
+  "confidence": 1,
+  "emission_date": "2026-09-20",
+  "period": "September 2026",
+  "emitting_company": "EDF",
+  "document_type": "facture",
+  "optional_fields": {},
+  "extracted_text": "E2E sample document 2026-09-20",
+  "created_at": "2026-01-01T00:00:00+00:00",
+  "updated_at": "2026-01-01T00:00:00+00:00",
+  "events": []
+}

automatic-pdf-renamer/tests/e2e/fixtures/baseline/pending/valid.pdf

@@ -0,0 +1,32 @@
+%PDF-1.4
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+2 0 obj
+<< /Type /Pages /Kids [3 0 R] /Count 1 >>
+endobj
+3 0 obj
+<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>
+endobj
+4 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>
+endobj
+5 0 obj
+<< /Length 61 >>
+stream
+BT /F1 12 Tf 72 720 Td (E2E sample document 2026-09-20) Tj ET
+endstream
+endobj
+xref
+0 6
+0000000000 65535 f 
+0000000009 00000 n 
+0000000058 00000 n 
+0000000115 00000 n 
+0000000241 00000 n 
+0000000311 00000 n 
+trailer
+<< /Size 6 /Root 1 0 R >>
+startxref
+422
+%%EOF

automatic-pdf-renamer/tests/e2e/requirements.txt

@@ -0,0 +1 @@
+pytest>=8.0,<9.0

automatic-pdf-renamer/tests/e2e/test_scenarios.py

@@ -0,0 +1,207 @@
+from __future__ import annotations
+
+import json
+import sqlite3
+from pathlib import Path
+from typing import Any
+
+from conftest import request_json
+
+
+def _sidecar(storage: Path, key: str) -> dict[str, Any]:
+    return json.loads((storage / key).with_suffix(".meta.json").read_text(encoding="utf-8"))
+
+
+def _database_rows(db_path: Path) -> list[dict[str, Any]]:
+    with sqlite3.connect(db_path) as connection:
+        connection.row_factory = sqlite3.Row
+        return [dict(row) for row in connection.execute("SELECT * FROM files ORDER BY sha256")]
+
+
+def _filesystem_snapshot(storage: Path) -> dict[str, bytes]:
+    return {
+        str(path.relative_to(storage)): path.read_bytes()
+        for path in storage.rglob("*") if path.is_file()
+    }
+
+
+def _assert_baseline(service: dict[str, Any], *, expected_count: int) -> None:
+    rows = _database_rows(service["db_path"])
+    assert len(rows) == expected_count
+    assert {row["sha256"] for row in rows} == ({"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "b6c81b0f56ce2237b8d121f1066ed699fc32cd18a587e337cb7273d63e862fa4"} if expected_count else set())
+    assert _sidecar(service["storage"], "inbox/empty.pdf")["sha256"] == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+    assert _sidecar(service["storage"], "pending/valid.pdf")["sha256"] == "b6c81b0f56ce2237b8d121f1066ed699fc32cd18a587e337cb7273d63e862fa4"
+
+
+def test_rebuild_and_read_only_requests(e2e_service: dict[str, Any]) -> None:
+    """Run an ordered action list and check persisted state after each action."""
+    service = e2e_service
+    base = service["base_url"]
+    storage = service["storage"]
+    original_files = _filesystem_snapshot(storage)
+
+    # Action 1: rebuild SQLite from authoritative sidecars.
+    status, result, _ = request_json(base, "POST", "/consistency/rebuild")
+    assert status == 200, result
+    assert result["status"] == "completed"
+    assert result["count"] == 2
+    _assert_baseline(service, expected_count=2)
+    assert _filesystem_snapshot(storage) == original_files
+
+    # Action 2: list documents.
+    status, items, headers = request_json(base, "GET", "/files")
+    assert status == 200, items
+    assert len(items) == 2
+    assert {key.lower(): value for key, value in headers.items()}.get("x-total-count") == "2"
+    _assert_baseline(service, expected_count=2)
+    assert _filesystem_snapshot(storage) == original_files
+
+    # Action 3: read document detail, including extracted text.
+    status, detail, _ = request_json(base, "GET", f"/files/b6c81b0f56ce2237b8d121f1066ed699fc32cd18a587e337cb7273d63e862fa4")
+    assert status == 200, detail
+    assert detail["current_key"] == "pending/valid.pdf"
+    assert detail["extracted_text"] == "E2E sample document 2026-09-20"
+    _assert_baseline(service, expected_count=2)
+    assert _filesystem_snapshot(storage) == original_files
+
+
+def test_missing_document_returns_404_without_mutation(e2e_service: dict[str, Any]) -> None:
+    service = e2e_service
+    request_status, result, _ = request_json(service["base_url"], "POST", "/consistency/rebuild")
+    assert request_status == 200, result
+    before = _filesystem_snapshot(service["storage"])
+
+    status, body, _ = request_json(service["base_url"], "GET", f"/files/{'f' * 64}")
+    assert status == 404, body
+    _assert_baseline(service, expected_count=2)
+    assert _filesystem_snapshot(service["storage"]) == before
+
+
+def test_pdf_endpoint_streams_original_bytes(e2e_service: dict[str, Any]) -> None:
+    import urllib.request
+
+    service = e2e_service
+    request_status, result, _ = request_json(service["base_url"], "POST", "/consistency/rebuild")
+    assert request_status == 200, result
+    original_pdf = (service["storage"] / "pending/valid.pdf").read_bytes()
+    before = _filesystem_snapshot(service["storage"])
+
+    with urllib.request.urlopen(
+        service["base_url"] + f"/files/b6c81b0f56ce2237b8d121f1066ed699fc32cd18a587e337cb7273d63e862fa4/pdf", timeout=10
+    ) as response:
+        assert response.status == 200
+        assert response.headers.get_content_type() == "application/pdf"
+        assert response.read() == original_pdf
+
+    _assert_baseline(service, expected_count=2)
+    assert _filesystem_snapshot(service["storage"]) == before
+
+def test_file_list_filters_and_search(e2e_service: dict[str, Any]) -> None:
+    service = e2e_service
+    base = service["base_url"]
+    status_code, rebuilt, _ = request_json(base, "POST", "/consistency/rebuild")
+    assert status_code == 200, rebuilt
+
+    status_code, processed, headers = request_json(base, "GET", "/files?status=processed")
+    assert status_code == 200, processed
+    assert len(processed) == 1
+    assert processed[0]["id"] == "b6c81b0f56ce2237b8d121f1066ed699fc32cd18a587e337cb7273d63e862fa4"
+    assert {key.lower(): value for key, value in headers.items()}.get("x-total-count") == "1"
+
+    status_code, by_company, _ = request_json(base, "GET", "/files?emitting_company=EDF")
+    assert status_code == 200, by_company
+    assert len(by_company) == 1
+    assert by_company[0]["emitting_company"] == "EDF"
+
+    status_code, by_text, _ = request_json(base, "GET", "/files?q=E2E%20sample%20document")
+    assert status_code == 200, by_text
+    assert any(item["id"] == "b6c81b0f56ce2237b8d121f1066ed699fc32cd18a587e337cb7273d63e862fa4" for item in by_text)
+
+
+def test_document_edit_updates_metadata_and_moves_pdf_safely(e2e_service: dict[str, Any]) -> None:
+    service = e2e_service
+    base = service["base_url"]
+    file_id = "b6c81b0f56ce2237b8d121f1066ed699fc32cd18a587e337cb7273d63e862fa4"
+    status_code, rebuilt, _ = request_json(base, "POST", "/consistency/rebuild")
+    assert status_code == 200, rebuilt
+
+    source_pdf = service["storage"] / "pending/valid.pdf"
+    original_bytes = source_pdf.read_bytes()
+    payload = {
+        "template_name": "default",
+        "period_format": "monthly",
+        "emitting_company": "EDF",
+        "document_type": "facture",
+        "emission_date": "2026-09-20",
+        "optional_fields": {"customer_number": "CUST-12345678"},
+    }
+    status_code, detail, _ = request_json(base, "POST", f"/files/{file_id}/edit", payload)
+    assert status_code == 200, detail
+    assert detail["status"] == "validated"
+    assert detail["emission_date"] == "2026-09-20"
+    assert detail["period"] == "Septembre 2026"
+    assert detail["current_key"].startswith("files/EDF/Septembre 2026/facture/2026-09-20_valid.pdf")
+    assert detail["optional_fields"] == {"customer_number": "CUST-12345678"}
+
+    destination = service["storage"] / detail["current_key"]
+    assert destination.read_bytes() == original_bytes
+    assert not source_pdf.exists()
+    assert not (service["storage"] / "pending/valid.meta.json").exists()
+
+    rows = _database_rows(service["db_path"])
+    matching = [row for row in rows if row["sha256"] == file_id]
+    assert len(matching) == 1
+    assert matching[0]["status"] == "validated"
+    assert matching[0]["current_key"] == detail["current_key"]
+    assert matching[0]["optional_fields"] == json.dumps({"customer_number": "CUST-12345678"}, separators=(",", ":")) or json.loads(matching[0]["optional_fields"]) == {"customer_number": "CUST-12345678"}
+
+    sidecar_path = service["storage"] / f"{detail['current_key']}.meta.json"
+    assert sidecar_path.exists()
+    sidecar = json.loads(sidecar_path.read_text(encoding="utf-8"))
+    assert sidecar["status"] == "validated"
+    assert sidecar["optional_fields"] == {"customer_number": "CUST-12345678"}
+    assert any(event.get("type") == "corrected" for event in sidecar.get("events", []))
+
+
+def test_configuration_can_be_read_and_invalid_update_is_rejected(e2e_service: dict[str, Any]) -> None:
+    base = e2e_service["base_url"]
+    status_code, original, _ = request_json(base, "GET", "/api/config")
+    assert status_code == 200, original
+    assert isinstance(original.get("content"), str)
+    assert '"templates"' in original["content"]
+
+    status_code, error, _ = request_json(base, "PUT", "/api/config", {"content": '{"templates":'})
+    assert status_code == 422, error
+
+    status_code, after, _ = request_json(base, "GET", "/api/config")
+    assert status_code == 200, after
+    assert after["content"] == original["content"]
+
+
+def test_configuration_valid_update_reloads_without_changing_content(e2e_service: dict[str, Any]) -> None:
+    base = e2e_service["base_url"]
+    status_code, original, _ = request_json(base, "GET", "/api/config")
+    assert status_code == 200, original
+
+    status_code, saved, _ = request_json(base, "PUT", "/api/config", {"content": original["content"]})
+    assert status_code == 200, saved
+    assert saved == {"saved": True, "reloaded": True}
+
+    status_code, current, _ = request_json(base, "GET", "/api/config")
+    assert status_code == 200, current
+    assert current["content"] == original["content"]
+
+
+def test_template_options_are_available_for_dashboard(e2e_service: dict[str, Any]) -> None:
+    base = e2e_service["base_url"]
+    status_code, templates, _ = request_json(base, "GET", "/templates")
+    assert status_code == 200, templates
+    assert {template["name"] for template in templates} >= {"default", "edf_invoice", "bank_statement"}
+
+    status_code, options, _ = request_json(base, "GET", "/api/document-options")
+    assert status_code == 200, options
+    assert "templates" in options
+    assert "period_formats" in options
+    assert "companies" in options
+    assert "document_types" in options
+