mirror of
https://github.com/napnap75/multiarch-docker-images.git
synced 2026-11-11 00:22:12 +01:00
Add configurable document columns and allow bulk metadata editing (#14)
This commit is contained in:
@@ -476,3 +476,23 @@ The following are implementation details, not reasons to block the API contract:
|
||||
- Exact SQLite full-text search tokenizer and phrase/prefix matching semantics.
|
||||
- Whether PDF byte-range requests are supported by the chosen S3 client/viewer.
|
||||
- Exact representation of technical processing diagnostics after retries are exhausted.
|
||||
|
||||
## 4.8 Configurable list and bulk-edit API contract (proposed)
|
||||
|
||||
The file-list API must expose a stable metadata modification timestamp (last_modified) distinct from emission_date. List results must support the existing filter/search/sort contract and a stable row identity for selection.
|
||||
|
||||
The frontend needs a server-side way to resolve all records matching the active filters, independently of the current page. Selection across filtered results must not be implemented by collecting only the currently loaded page's IDs. The exact transport may be an explicit select_all_matching operation or a bulk endpoint accepting the validated filter expression; it must reuse the same filter semantics as the list endpoint.
|
||||
|
||||
A bulk-edit endpoint is required under /api/v1/files (exact route to be aligned with the existing router implementation). Its request contract must express:
|
||||
- an explicit set of document IDs or an all-matching selection plus the active filters;
|
||||
- explicit operations for emitting_company, document_type, and keyed additional_fields;
|
||||
- for each field, no-op/unchanged, set/replace, or clear (clear only for optional fields);
|
||||
- an idempotency key or equivalent retry-safe mechanism for partial outcomes.
|
||||
|
||||
The response must include requested count, succeeded count, failed count, and per-document results with stable identity, status, and actionable validation/technical error details. Invalid input must never be treated as a successful no-op. Bulk requests must be bounded or processed as a background job when size requires it, with progress/status available to the UI.
|
||||
|
||||
The backend must apply the same validation as single-document edits. It recalculates period and destination key server-side, writes authoritative sidecars before synchronizing SQLite, uses the established safe object-move sequence and collision policy, and appends a history event per successful document. period, current_key, status, and timestamps are server-managed and cannot be supplied as arbitrary client values.
|
||||
|
||||
## Additional list-filter contract
|
||||
|
||||
The document list accepts an `emission_date` filter and one filter per configured additional field. Additional-field filters use the query parameter prefix `additional_field__` followed by the field's stable key (for example, `additional_field__account=123`). Both are applied server-side using the same semantics for paginated list requests and “select all matching” bulk operations. The UI renders a filter control directly beneath every visible emission-date or additional-field column, and changing visible columns preserves any active filter values.
|
||||
|
||||
@@ -28,6 +28,7 @@ class FileListItem(BaseModel):
|
||||
emission_date: str | None = None
|
||||
confidence: float | None = None
|
||||
updated_at: str | None = None
|
||||
optional_fields: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class FileDetail(FileListItem):
|
||||
@@ -58,6 +59,15 @@ class DocumentEditRequest(BaseModel):
|
||||
optional_fields: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class BulkEditRequest(BaseModel):
|
||||
"""Apply explicit metadata changes to selected documents or filtered results."""
|
||||
|
||||
ids: list[str] = Field(default_factory=list)
|
||||
all_matching: bool = False
|
||||
filters: dict[str, Any] = Field(default_factory=dict)
|
||||
changes: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class ConfigUpdateRequest(BaseModel):
|
||||
"""JSONC configuration text submitted by the dashboard."""
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
|
||||
|
||||
from app.config import TemplateRegistry, load_templates
|
||||
from app.db import FileRecord, SQLiteFileRepository
|
||||
from app.period import format_period_from_emission_date
|
||||
from app.period import format_period_from_emission_date, get_period_key_values
|
||||
from app.pipeline import DocumentProcessor
|
||||
from app.storage import StorageBackend
|
||||
from app.model_manager import list_models, train_all_models
|
||||
@@ -26,6 +26,7 @@ from .models import (
|
||||
FileListItem,
|
||||
ConfigUpdateRequest,
|
||||
DocumentEditRequest,
|
||||
BulkEditRequest,
|
||||
HealthResponse,
|
||||
ValidationRequest,
|
||||
ValidationResponse,
|
||||
@@ -92,6 +93,7 @@ def _as_file_item(file_id: str, metadata: dict[str, Any]) -> FileListItem:
|
||||
emission_date=metadata.get("emission_date"),
|
||||
confidence=metadata.get("confidence"),
|
||||
updated_at=metadata.get("updated_at"),
|
||||
optional_fields=dict(metadata.get("optional_fields") or {}),
|
||||
)
|
||||
|
||||
|
||||
@@ -119,15 +121,21 @@ async def list_files(
|
||||
period: str | None = None,
|
||||
confidence: str | None = None,
|
||||
updated_at: str | None = None,
|
||||
emission_date: str | None = None,
|
||||
sort_by: str = "updated_at",
|
||||
sort_order: str = "desc",
|
||||
page: int = 1,
|
||||
) -> list[FileListItem]:
|
||||
"""List documents and report filtered and global counts in response headers."""
|
||||
logger.debug("API GET /files: user=%s page=%s sort_by=%s sort_order=%s", (current_user or {}).get("username", "anonymous"), page, sort_by, sort_order)
|
||||
logger.debug("API GET /files: active filters=%s", {key: value for key, value in {"status": status, "template": template, "period_format": period_format, "q": q, "current_key": current_key, "emitting_company": emitting_company, "document_type": document_type, "period": period, "confidence": confidence, "updated_at": updated_at}.items() if value is not None})
|
||||
storage = _storage(request)
|
||||
db = _db(request)
|
||||
additional_fields = {
|
||||
key.removeprefix("additional_field__"): value
|
||||
for key, value in request.query_params.items()
|
||||
if key.startswith("additional_field__") and value
|
||||
}
|
||||
logger.debug("API GET /files: active filters=%s", {key: value for key, value in {"status": status, "template": template, "period_format": period_format, "q": q, "current_key": current_key, "emitting_company": emitting_company, "document_type": document_type, "period": period, "confidence": confidence, "updated_at": updated_at, "emission_date": emission_date, **{f"additional_field__{k}": v for k, v in additional_fields.items()}}.items() if value is not None})
|
||||
filters = {
|
||||
"status": status,
|
||||
"template": template,
|
||||
@@ -139,6 +147,8 @@ async def list_files(
|
||||
"period_format": period_format,
|
||||
"confidence": confidence,
|
||||
"updated_at": updated_at,
|
||||
"emission_date": emission_date,
|
||||
"additional_fields": additional_fields,
|
||||
}
|
||||
|
||||
if db.count_files() == 0:
|
||||
@@ -175,6 +185,7 @@ async def list_files(
|
||||
emission_date=record.emission_date,
|
||||
confidence=record.confidence,
|
||||
updated_at=record.updated_at,
|
||||
optional_fields=dict(record.optional_fields or {}),
|
||||
)
|
||||
for record in records
|
||||
]
|
||||
@@ -374,6 +385,9 @@ async def get_document_model_suggestions(
|
||||
"additional_fields": ", ".join(str(value) for value in optional_fields.values()),
|
||||
"original_filename": (record.original_key or "").rsplit("/", 1)[-1],
|
||||
})
|
||||
pattern_values.update(
|
||||
get_period_key_values(emission_date, period_config.to_dict())
|
||||
)
|
||||
for _ in range(20):
|
||||
try:
|
||||
calculated_key = key_pattern.format(**pattern_values).strip("/")
|
||||
@@ -507,6 +521,9 @@ async def edit_file(
|
||||
"additional_fields": ", ".join(updated_optional_fields.values()),
|
||||
"original_filename": record.original_key.rsplit("/", 1)[-1],
|
||||
})
|
||||
pattern_values.update(
|
||||
get_period_key_values(payload.emission_date, period_config.to_dict())
|
||||
)
|
||||
for _ in range(20):
|
||||
try:
|
||||
formatted_key = key_pattern.format(**pattern_values).strip("/")
|
||||
@@ -594,7 +611,8 @@ async def edit_file(
|
||||
new_values["optional_fields"] = dict(updated_optional_fields)
|
||||
sidecar_metadata.update(new_values)
|
||||
sidecar_metadata["optional_fields"] = updated_optional_fields
|
||||
sidecar_metadata["status"] = "validated"
|
||||
preserve_status = bool(getattr(request.state, "preserve_document_status", False))
|
||||
sidecar_metadata["status"] = record.status if preserve_status else "validated"
|
||||
sidecar_metadata["updated_at"] = updated_at
|
||||
events = sidecar_metadata.get("events")
|
||||
if not isinstance(events, list):
|
||||
@@ -624,7 +642,8 @@ async def edit_file(
|
||||
record.period = period
|
||||
record.current_key = new_pdf_key
|
||||
record.optional_fields = updated_optional_fields
|
||||
record.status = "validated"
|
||||
if not preserve_status:
|
||||
record.status = "validated"
|
||||
record.updated_at = updated_at
|
||||
db.upsert(record)
|
||||
|
||||
@@ -636,6 +655,81 @@ async def edit_file(
|
||||
return await get_file(request, record.id, current_user=current_user)
|
||||
|
||||
|
||||
@router.post("/files/bulk-edit")
|
||||
async def bulk_edit_files(
|
||||
request: Request,
|
||||
payload: BulkEditRequest,
|
||||
current_user: dict[str, str] | None = Depends(get_current_user),
|
||||
) -> dict[str, Any]:
|
||||
"""Apply company, document-type, and additional-field edits to selected files."""
|
||||
db = _db(request)
|
||||
if not payload.changes:
|
||||
raise HTTPException(status_code=422, detail="At least one explicit change is required")
|
||||
allowed = {"template_name", "period_format", "emitting_company", "document_type", "emission_date", "additional_fields"}
|
||||
unknown = set(payload.changes) - allowed
|
||||
if unknown:
|
||||
raise HTTPException(status_code=422, detail=f"Unsupported bulk fields: {', '.join(sorted(unknown))}")
|
||||
if payload.all_matching:
|
||||
if payload.ids:
|
||||
raise HTTPException(status_code=422, detail="Specify ids or all_matching, not both")
|
||||
filters = {key: value for key, value in payload.filters.items() if key in {
|
||||
"status", "template", "period_format", "q", "current_key", "emitting_company",
|
||||
"document_type", "period", "confidence", "updated_at", "emission_date"
|
||||
}}
|
||||
additional_fields = {
|
||||
key.removeprefix("additional_field__"): value
|
||||
for key, value in payload.filters.items()
|
||||
if key.startswith("additional_field__") and value
|
||||
}
|
||||
records = db.list_files(**filters, additional_fields=additional_fields, page=1, page_size=100000)
|
||||
else:
|
||||
if not payload.ids:
|
||||
raise HTTPException(status_code=422, detail="Select at least one document")
|
||||
records = [record for file_id in dict.fromkeys(payload.ids) if (record := db.get_by_id(file_id)) is not None]
|
||||
if not records:
|
||||
raise HTTPException(status_code=404, detail="No matching documents found")
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
# Bulk metadata edits must not implicitly validate documents.
|
||||
request.state.preserve_document_status = True
|
||||
for record in records:
|
||||
try:
|
||||
if not record.template_name or not record.period_format or not record.emission_date:
|
||||
raise HTTPException(status_code=422, detail="Document is missing template, period format, or emission date")
|
||||
optional_fields = dict(record.optional_fields or {})
|
||||
field_changes = payload.changes.get("additional_fields", {})
|
||||
if field_changes is not None:
|
||||
if not isinstance(field_changes, dict):
|
||||
raise HTTPException(status_code=422, detail="additional_fields must be an object")
|
||||
for name, value in field_changes.items():
|
||||
if not str(name).strip():
|
||||
raise HTTPException(status_code=422, detail="Additional-field names cannot be empty")
|
||||
if value is None or value == {"operation": "clear"}:
|
||||
optional_fields.pop(str(name), None)
|
||||
elif isinstance(value, dict) and value.get("operation") == "set":
|
||||
optional_fields[str(name)] = value.get("value", "")
|
||||
else:
|
||||
optional_fields[str(name)] = value
|
||||
edit_payload = DocumentEditRequest(
|
||||
template_name=str(payload.changes.get("template_name", record.template_name)),
|
||||
period_format=str(payload.changes.get("period_format", record.period_format)),
|
||||
emitting_company=str(payload.changes.get("emitting_company", record.emitting_company or "")),
|
||||
document_type=str(payload.changes.get("document_type", record.document_type or "")),
|
||||
emission_date=payload.changes.get("emission_date", record.emission_date),
|
||||
optional_fields=optional_fields,
|
||||
)
|
||||
updated = await edit_file(request, record.id, edit_payload, current_user=current_user)
|
||||
results.append({"id": record.id, "success": True, "status": updated.status, "current_key": updated.current_key})
|
||||
except HTTPException as error:
|
||||
results.append({"id": record.id, "success": False, "status_code": error.status_code, "error": error.detail})
|
||||
except Exception as error:
|
||||
logger.exception("Bulk edit failed for file %s", record.id)
|
||||
results.append({"id": record.id, "success": False, "status_code": 500, "error": str(error)})
|
||||
|
||||
succeeded = sum(1 for item in results if item["success"])
|
||||
return {"requested": len(results), "succeeded": succeeded, "failed": len(results) - succeeded, "results": results}
|
||||
|
||||
|
||||
@router.post("/files/{file_id}/validate", response_model=ValidationResponse)
|
||||
async def validate_file(
|
||||
request: Request,
|
||||
|
||||
@@ -246,6 +246,8 @@ class SQLiteFileRepository:
|
||||
period: str | None = None,
|
||||
confidence: str | None = None,
|
||||
updated_at: str | None = None,
|
||||
emission_date: str | None = None,
|
||||
additional_fields: dict[str, str] | None = None,
|
||||
) -> tuple[list[str], list[Any]]:
|
||||
conditions: list[str] = []
|
||||
params: list[Any] = []
|
||||
@@ -265,11 +267,17 @@ class SQLiteFileRepository:
|
||||
("document_type", document_type),
|
||||
("period", period),
|
||||
("updated_at", updated_at),
|
||||
("emission_date", emission_date),
|
||||
("period_format", period_format),
|
||||
):
|
||||
if value:
|
||||
conditions.append(f"LOWER({column}) LIKE ?")
|
||||
params.append(f"%{value.lower()}%")
|
||||
for field_name, field_value in (additional_fields or {}).items():
|
||||
if field_value:
|
||||
json_path = '$."' + field_name.replace('"', '\\"') + '"'
|
||||
conditions.append("LOWER(COALESCE(json_extract(optional_fields, ?), '')) LIKE ?")
|
||||
params.extend([json_path, f"%{field_value.lower()}%"])
|
||||
if confidence:
|
||||
confidence_filter = confidence.strip().removesuffix("%")
|
||||
if confidence_filter:
|
||||
@@ -296,6 +304,8 @@ class SQLiteFileRepository:
|
||||
period: str | None = None,
|
||||
confidence: str | None = None,
|
||||
updated_at: str | None = None,
|
||||
emission_date: str | None = None,
|
||||
additional_fields: dict[str, str] | None = None,
|
||||
sort_by: str = "updated_at",
|
||||
sort_order: str = "desc",
|
||||
page: int = 1,
|
||||
@@ -312,6 +322,8 @@ class SQLiteFileRepository:
|
||||
period=period,
|
||||
confidence=confidence,
|
||||
updated_at=updated_at,
|
||||
emission_date=emission_date,
|
||||
additional_fields=additional_fields,
|
||||
)
|
||||
|
||||
query = "SELECT * FROM files"
|
||||
@@ -350,6 +362,8 @@ class SQLiteFileRepository:
|
||||
period: str | None = None,
|
||||
confidence: str | None = None,
|
||||
updated_at: str | None = None,
|
||||
emission_date: str | None = None,
|
||||
additional_fields: dict[str, str] | None = None,
|
||||
) -> int:
|
||||
conditions, params = self._file_conditions(
|
||||
status=status,
|
||||
@@ -362,6 +376,8 @@ class SQLiteFileRepository:
|
||||
period=period,
|
||||
confidence=confidence,
|
||||
updated_at=updated_at,
|
||||
emission_date=emission_date,
|
||||
additional_fields=additional_fields,
|
||||
)
|
||||
query = "SELECT COUNT(*) FROM files"
|
||||
if conditions:
|
||||
|
||||
@@ -561,6 +561,27 @@ def format_period_from_emission_date(
|
||||
raise ValueError(f"Unknown placeholder '{missing}' in format template: {format_template}")
|
||||
|
||||
|
||||
def get_period_key_values(
|
||||
emission_date: str | date,
|
||||
period_format: dict[str, Any],
|
||||
) -> dict[str, str]:
|
||||
"""Return period placeholders, preserving the source date's day of month."""
|
||||
separator = "\x1f"
|
||||
values = format_period_from_emission_date(
|
||||
emission_date,
|
||||
{**period_format, "format": separator.join(
|
||||
("{year}", "{month}", "{month_name}", "{day}")
|
||||
)},
|
||||
).split(separator)
|
||||
year, month, month_name, _ = values
|
||||
return {
|
||||
"year": year,
|
||||
"month": month,
|
||||
"month_name": month_name,
|
||||
"day": f"{_parse_date_string(emission_date).day:02d}",
|
||||
}
|
||||
|
||||
|
||||
def format_month_year(month: int, year: int, use_french: bool = True) -> str:
|
||||
"""Format month and year as a human-readable string.
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ from .config import (
|
||||
from .extractor import extract_document_text, is_text_empty, TextExtractionError
|
||||
from .period import (
|
||||
format_period_from_emission_date,
|
||||
get_period_key_values,
|
||||
)
|
||||
from .sidecar import Sidecar, SidecarStatus, build_sidecar
|
||||
from .ml import LocalMLPipeline, ModelUnavailableError, AmbiguousPredictionError
|
||||
@@ -225,18 +226,21 @@ class DocumentProcessor:
|
||||
key_pattern = self.registry.get_key_pattern(template_name)
|
||||
if key_pattern:
|
||||
try:
|
||||
final_key = self._format_key_pattern(
|
||||
key_pattern,
|
||||
company=company,
|
||||
document_type=doc_type,
|
||||
emission_date=emission_date,
|
||||
period=period,
|
||||
template_name=template_name,
|
||||
period_format=period_format,
|
||||
emitting_company=company,
|
||||
original_filename=original_key.split("/")[-1],
|
||||
key_values = {
|
||||
"company": company,
|
||||
"document_type": doc_type,
|
||||
"emission_date": emission_date,
|
||||
"period": period,
|
||||
"template_name": template_name,
|
||||
"period_format": period_format,
|
||||
"emitting_company": company,
|
||||
"original_filename": original_key.split("/")[-1],
|
||||
**optional_fields,
|
||||
)
|
||||
**get_period_key_values(
|
||||
emission_date, period_format_config.to_dict()
|
||||
),
|
||||
}
|
||||
final_key = self._format_key_pattern(key_pattern, **key_values)
|
||||
except Exception as e:
|
||||
result.errors.append(f"Failed to format key pattern: {e}")
|
||||
result.status = SidecarStatus.PENDING.value
|
||||
@@ -289,6 +293,14 @@ class DocumentProcessor:
|
||||
|
||||
return result
|
||||
|
||||
def _format_key_pattern(self, pattern: str, **values: Any) -> str:
|
||||
"""Format a template key pattern with document and period values."""
|
||||
try:
|
||||
return pattern.format(**values)
|
||||
except KeyError as exc:
|
||||
placeholder = str(exc).strip("'")
|
||||
raise ValueError(f"Unknown placeholder '{placeholder}' in key pattern") from exc
|
||||
|
||||
def _extract_text(self, pdf_bytes: bytes) -> str | None:
|
||||
"""Extract text from every page of the PDF.
|
||||
|
||||
|
||||
@@ -405,3 +405,11 @@ Every epic maps to a component: ingestion (jobs/poller), extraction and classifi
|
||||
### Local ML inference
|
||||
|
||||
The runtime loads trusted, locally stored model artifacts from `MODEL_DIR`. The analysis order is mandatory: (1) predict `emitting_company` and `document_type` from extracted text; (2) extract `emission_date` using the configured `date_extraction` regexes only, selecting the earliest valid match in document order (no date ML model); (3) predict only the applicable additional-field groups from `emitting_company` and `document_type`; (4) extract each additional-field value from the full text using one shared token-level model; (5) predict `template_name` and `period_format` from `emitting_company`, `document_type`, and additional fields; and (6) calculate `period` once from the emission date and final period format, and calculate `current_key` from the template key pattern and final values. Training labels come from validated sidecar values, and additional-field values are trained only when they appear in the saved extracted text. Predictions are confidence-gated; missing or uncertain required predictions route to `pending`. Templates and period-format definitions provide calculation/formatting rules, not hard-coded classification mappings.
|
||||
|
||||
## Dashboard list and bulk-edit architecture changes
|
||||
|
||||
The list remains a SQLite-backed query view over sidecar-authoritative metadata. Add/expose a stable metadata last_modified timestamp and make the list's column catalogue describe both standard fields and configured additional fields (stable key, display label, value type, editability). The default visible set is Status, Document filename, Company, Document type, Period, Last modified; user column preferences are presentation state and must not be written into document sidecars.
|
||||
|
||||
Selection is query-based rather than DOM/page-based. The API must resolve all records matching the same validated filter specification used by list/search, so “all filtered” works across pagination without shipping an unbounded list of IDs from the browser. A bulk operation carries explicit field operations and an idempotency token. The service validates each document independently, reuses the single-document validation and rename services, and returns durable per-document outcomes.
|
||||
|
||||
For every successful document, persist the sidecar and event history before updating SQLite. Recompute derived period and key from changed source fields. If a key changes, preserve the copy → sidecar update → source deletion ordering and collision suffix rules. A failed item must leave its authoritative metadata in a recoverable state and be reported independently; successful items must be recognizable on retry. Keep bulk operations bounded, or run them as tracked background jobs when the request exceeds a safe synchronous size.
|
||||
|
||||
@@ -91,3 +91,13 @@ Single user (personal use), technically proficient. Nominal flow:
|
||||
- Partial uploads (Nextcloud chunked uploads): poll-only keys with a stable size across two listing cycles and a `.part`/`.~lock` exclusion list.
|
||||
- Sidecar/object drift (sidecar missing or out of sync with SQLite): sidecars win; a consistency job rebuilds SQLite and flags orphaned or missing sidecars.
|
||||
- Garage version of S3 API: pin a Garage release and run a compatibility spike (listing, multipart, copy-object, versioning).
|
||||
|
||||
## Dashboard list usability and bulk editing (change proposal)
|
||||
|
||||
The home page is a configurable document table. Its default visible columns, in order, are **Status**, **Document** (current filename), **Company**, **Document type**, **Period**, and **Last modified**. “Last modified” is the document metadata's latest modification timestamp, not the PDF emission date or filesystem creation time.
|
||||
|
||||
Users can show or hide available columns, including any configured additional field. Column configuration affects presentation only: it must not alter metadata or filters. Additional-field columns are labelled from their configured field definitions and render missing values clearly. The list retains existing filtering, search, sorting, and consistency indicators.
|
||||
|
||||
Each row has a selection control. Users can select individual visible rows, select all rows on the current page, or select **all rows matching the active filters** across pagination. The UI must state which selection mode is active and the number of affected documents. A bulk-edit action applies explicitly chosen changes to the selected set: company, document type, and configured additional fields (set, replace, or clear a value). Unchanged fields are preserved. The user reviews the proposed changes and affected count before confirmation. Validation failures and per-document outcomes are reported; one invalid document must not silently hide failures for the others.
|
||||
|
||||
Bulk edits use the existing metadata and validation rules, recompute derived period and destination key when relevant fields change, and safely rename/move objects when the target key changes. Sidecars remain authoritative; SQLite is synchronized after durable sidecar updates. Every change is auditable in the document event history. The feature must not silently invent company/type values or overwrite colliding objects.
|
||||
|
||||
@@ -575,3 +575,31 @@ Completes the remaining Epic 4 capabilities and Epic 5, including consistency re
|
||||
## 7. Open Decisions to Resolve Before the Affected Stories
|
||||
|
||||
2. **ML field extraction:** specify which field groups use ML in v1 and what evaluation threshold is required before enabling the shared extractor. Classification and deterministic validation must remain usable independently.
|
||||
|
||||
## New dashboard epic — Configurable list and bulk metadata editing
|
||||
|
||||
### Story: Configure visible columns
|
||||
As the owner, I want to choose which metadata columns appear on the home page, including configured additional fields, so that I can tailor the table to the task at hand.
|
||||
|
||||
Acceptance criteria:
|
||||
- Default visible order is Status, Document (filename), Company, Document type, Period, Last modified.
|
||||
- Users can add/remove available columns, including additional fields identified by stable field keys.
|
||||
- Empty values are rendered consistently; hiding a column does not delete or change its data.
|
||||
|
||||
### Story: Select documents
|
||||
As the owner, I want to select rows or every document matching the current filters, including across pagination, so that I can perform a batch operation efficiently.
|
||||
|
||||
Acceptance criteria:
|
||||
- Each row has a selection control; page selection and all-filtered selection are distinct.
|
||||
- The UI displays the selected count and clearly communicates the selection scope.
|
||||
- Filter changes cannot leave a misleading stale selection.
|
||||
|
||||
### Story: Bulk edit metadata
|
||||
As the owner, I want to change company, document type, and additional-field values for multiple selected documents in one operation, so that I can correct repeated classification mistakes efficiently.
|
||||
|
||||
Acceptance criteria:
|
||||
- Each editable field defaults to unchanged; additional fields support set/replace and clear.
|
||||
- A confirmation step summarizes the operation and affected count.
|
||||
- The server validates each document using the single-document rules, recomputes derived period and target key, and safely handles rename collisions.
|
||||
- Sidecars remain authoritative, SQLite is synchronized, and an event is recorded for every successful change.
|
||||
- Partial failures are reported per document; successful documents are not silently reprocessed on retry.
|
||||
|
||||
@@ -206,3 +206,28 @@ SQLite replica (rebuilt from sidecars):
|
||||
### ML-only inference contract
|
||||
|
||||
The runtime uses local model artifacts for emitting-company, document-type and emission-date extraction; additional-field-group selection from emitting company and document type; shared token-level additional-field span extraction; and independent `template_name` and `period_format` prediction from emitting company and document type. Configuration may define template key patterns and period arithmetic parameters, but it must not select classification outcomes. Missing models, invalid predictions, or confidence below `CONFIDENCE_THRESHOLD` route the document to `pending`. The `period` and final storage key remain deterministic calculations after model inference.
|
||||
|
||||
## Change requirements — configurable document list and bulk editing
|
||||
|
||||
### FR6.5 — Default columns
|
||||
The home page MUST show these columns by default and in this order: status, document (current filename), company (emitting company), document type, period, and last modified. “Last modified” MUST be a persisted metadata modification timestamp and MUST NOT be confused with emission date or object creation time.
|
||||
|
||||
### FR6.6 — Configurable columns
|
||||
Users MUST be able to add and remove columns without changing the underlying records or filters. Available columns include all common/list fields and configured additional fields. Column IDs must be stable (additional fields use their field key, not their display label). A field with no value displays a consistent empty marker. The default column set is used when no user preference exists; if preferences are persisted, unavailable fields are safely ignored.
|
||||
|
||||
### FR6.7 — Row and filtered-result selection
|
||||
Users MUST be able to select individual rows, clear selection, select all rows on the current page, and select all records matching the current filters across pagination. “Select all filtered” MUST use the server-side filter query rather than only the rows loaded in the browser. The UI MUST distinguish page selection from all-filtered selection and show the affected count. Changing filters MUST invalidate or explicitly reconcile a prior selection.
|
||||
|
||||
### FR6.8 — Bulk metadata edit
|
||||
Users MUST be able to apply one or more changes to the selected documents: emitting company, document type, and any configured additional field. Each field in the bulk form supports “leave unchanged” as the default; additional fields can be set/replaced or explicitly cleared. No field changes unless the user explicitly chooses an operation. Show a review/confirmation step with selection scope, count, and proposed changes.
|
||||
|
||||
### FR6.9 — Validation, persistence, and feedback
|
||||
Bulk changes MUST use the same validation and business rules as the single-document workflow. Company and document type must be configured/allowed values. Recompute derived period and destination key whenever relevant source fields change; period remains server-calculated and read-only. If a key changes, use the existing crash-safe copy → durable sidecar → delete-source sequence, collision policy, and sidecar-first SQLite synchronization. Preserve unchanged fields, record an event per document, and return per-document success/failure details. Partial failures must be visible and retryable without reapplying successful items unintentionally.
|
||||
|
||||
### Acceptance criteria
|
||||
- Fresh users see exactly the six default columns above in the specified order.
|
||||
- A configured additional field can be added as a column and removed again.
|
||||
- “Select all filtered” includes matching records beyond the current page and excludes records outside the active filters.
|
||||
- Bulk editing can change company/type and set, replace, or clear an additional field while preserving all other values.
|
||||
- Derived period/key are recomputed where necessary; invalid edits do not corrupt sidecars, PDFs, or the SQLite replica.
|
||||
- The UI reports the number of selected records and per-document outcomes.
|
||||
|
||||
@@ -277,3 +277,7 @@ This ensures consistency between the import script and the main service.
|
||||
## License
|
||||
|
||||
Same as the main project. See [../LICENSE](../LICENSE).
|
||||
|
||||
## Dashboard list and bulk editing
|
||||
|
||||
The dashboard's configurable columns and bulk metadata editing are API/UI features, not standalone scripts. Script authors must preserve the existing sidecar-source-of-truth model: any script that changes company, document type, or additional fields must use the application's validation and safe rename flow, recompute derived period/key values, append event history, and synchronize SQLite only after the sidecar is durable. Do not treat user-visible column preferences as document metadata.
|
||||
|
||||
@@ -166,3 +166,46 @@ Once implemented, the repository should provide a single documented command to r
|
||||
The harness should never point at a developer's normal data directory or a production storage endpoint. It should fail early if its configuration appears to target a non-test database or external object storage.
|
||||
|
||||
Until the harness is implemented, this document is a design guide rather than a runnable command reference. See [API.md](API.md) for the API contract and [arch.md](arch.md) for the architecture overview.
|
||||
|
||||
## Additional test requirements — configurable columns and bulk editing
|
||||
|
||||
### Home-page columns
|
||||
- Assert the six default columns and their exact order: status, document filename, company, document type, period, last modified.
|
||||
- Assert that adding/removing a configured additional-field column changes only the visible table configuration, not document metadata.
|
||||
- Assert stable column IDs and a consistent rendering contract for empty values.
|
||||
|
||||
### Selection semantics
|
||||
- Select one row and multiple rows and verify the selected IDs.
|
||||
- Select the current page and verify that it does not claim to select every filtered record.
|
||||
- Select all filtered records across multiple pages; assert matching records outside the current page are included and nonmatching records are excluded.
|
||||
- Change or clear filters and verify stale selections are cleared or explicitly reconciled.
|
||||
|
||||
### Bulk edits
|
||||
- Change company and document type for multiple records; preserve every field not explicitly changed.
|
||||
- Set, replace, and clear an additional field; reject attempts to clear a required/common field.
|
||||
- Validate invalid configured values and verify that each failure is reported without corrupting the record.
|
||||
- When relevant values change, assert period/key recomputation and safe rename/collision handling.
|
||||
- Inject storage and SQLite-sync failures to assert sidecar-first consistency and recoverability.
|
||||
- Retry a partially successful operation and verify successful items are not applied twice and outcomes remain explicit.
|
||||
- Assert per-document event history, counts, and error details in the response.
|
||||
|
||||
## Additional filter-control tests
|
||||
|
||||
- When the emission-date column is visible, render a filter control beneath it and verify the entered date is sent as `emission_date` and filters the list.
|
||||
- When an additional-field column is visible, render a filter control beneath that column and verify its value is sent as `additional_field__<stable-field-key>`.
|
||||
- Verify additional-field filtering uses the stored JSON field value and does not match a different field with a similar name.
|
||||
- Verify date/additional-field filters remain active when the column picker is changed, and “select all filtered” bulk edits respect these filters across pagination.
|
||||
|
||||
## Bulk edit form interaction requirements
|
||||
|
||||
- The first table-header checkbox is the sole checkbox for selecting currently loaded rows; do not render a duplicate “select loaded rows” checkbox above the table.
|
||||
- Bulk edit uses the same option lists as `document.html` for template, period format, company, document type, and additional-field names/values.
|
||||
- Leaving a field on “Keep unchanged” must preserve its value on every selected document.
|
||||
- The user can add multiple additional-field rows, choose an existing field/value, clear a field explicitly, and remove a pending row before applying.
|
||||
- The confirmation summarizes the target count and all requested changes before submitting.
|
||||
- The server applies changes through the same single-document edit path, including derived period/key recalculation and collision handling.
|
||||
|
||||
## Bulk-edit status preservation
|
||||
|
||||
- Bulk metadata edits must preserve each document's existing status (for example, `processed`, `validated`, or `unknown`) in both SQLite and the sidecar metadata.
|
||||
- Editing a single document through the normal edit endpoint may continue to mark it as `validated`; the bulk-edit endpoint must not implicitly validate documents.
|
||||
|
||||
@@ -24,3 +24,15 @@ Fixtures are checked-in immutable inputs under `tests/e2e/fixtures/baseline/`. T
|
||||
## Safety
|
||||
|
||||
The suite always overrides storage and SQLite paths with paths inside its temporary workspace, and sets `APP_STORAGE_BACKEND=file`. It does not accept a production base URL or reuse a developer database. Do not modify the fixture source during a run.
|
||||
|
||||
## Additional E2E scenarios: configurable columns and bulk operations
|
||||
|
||||
The E2E suite should add API-level scenarios for the bulk-edit contract using isolated local storage and disposable SQLite state. The browser column chooser and row-selection interactions require frontend/browser tests; they are not covered by API-only E2E tests.
|
||||
|
||||
Required scenarios:
|
||||
- List records with last_modified and verify it is distinct from emission_date.
|
||||
- Bulk-edit company/type and an additional field; inspect sidecars and SQLite after completion.
|
||||
- Select all matching filters across pagination and prove nonmatching records remain unchanged.
|
||||
- Verify validation errors and per-document partial outcomes.
|
||||
- Force a destination collision and a storage failure; verify no source object is lost and sidecar/SQLite state remains recoverable.
|
||||
- Retry a partially successful request using the same idempotency token and verify duplicate mutations are not introduced.
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
"""Tests for asynchronous, confidence-gated document suggestions."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import unittest
|
||||
from datetime import date
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
from app.api import routes
|
||||
from app.api.models import DocumentEditRequest
|
||||
from app.ml import AmbiguousPredictionError, LocalMLPipeline
|
||||
|
||||
|
||||
@@ -32,7 +35,7 @@ class FakeRegistry:
|
||||
return SimpleNamespace(to_dict=lambda: {})
|
||||
|
||||
def get_key_pattern(self, name):
|
||||
return "example/{company}/{period}/{document_type}"
|
||||
return "example/{year}/{month}/{month_name}/{day}/{company}/{period}/{document_type}"
|
||||
|
||||
|
||||
class FakeMLPipeline(LocalMLPipeline):
|
||||
@@ -140,7 +143,10 @@ class DocumentSuggestionTests(unittest.TestCase):
|
||||
self.assertEqual(suggestions["template_name"]["value"], "invoice-template")
|
||||
self.assertEqual(suggestions["period_format"]["value"], "monthly")
|
||||
self.assertEqual(suggestions["period"]["value"], "2026-09")
|
||||
self.assertEqual(suggestions["current_key"]["value"], "files/example/Edited Company/2026-09/invoice.pdf")
|
||||
self.assertEqual(
|
||||
suggestions["current_key"]["value"],
|
||||
"files/example/2026/09/Septembre/20/Edited Company/2026-09/invoice.pdf",
|
||||
)
|
||||
|
||||
def test_derived_values_use_form_values_not_model_predictions(self):
|
||||
class ConflictingPipeline(FakeMLPipeline):
|
||||
@@ -303,6 +309,67 @@ class DocumentSuggestionTests(unittest.TestCase):
|
||||
self.assertEqual(suggestions["template_name"]["value"], "invoice-template")
|
||||
self.assertEqual(suggestions["period_format"]["value"], "monthly")
|
||||
|
||||
def test_edit_route_uses_computed_period_placeholders_in_storage_key(self):
|
||||
record = SimpleNamespace(
|
||||
id="document-1",
|
||||
sha256="document-1",
|
||||
current_key="pending/original.pdf",
|
||||
original_key="pending/original.pdf",
|
||||
template_name="old-template",
|
||||
period_format="monthly",
|
||||
emitting_company="Old Company",
|
||||
document_type="receipt",
|
||||
emission_date="2026-09-20",
|
||||
period="September 2026",
|
||||
optional_fields={},
|
||||
status="pending",
|
||||
confidence=None,
|
||||
updated_at="2026-09-20T00:00:00+00:00",
|
||||
created_at="2026-09-20T00:00:00+00:00",
|
||||
extracted_text="document text",
|
||||
)
|
||||
metadata = json.dumps({"sha256": record.sha256, "current_key": record.current_key})
|
||||
storage = Mock()
|
||||
storage.get_object.return_value = metadata
|
||||
storage.object_exists.side_effect = lambda key: key == record.current_key
|
||||
database = Mock()
|
||||
database.get_by_id.return_value = record
|
||||
registry = Mock()
|
||||
registry.get_template.return_value = object()
|
||||
registry.get_period_format_config.return_value = SimpleNamespace(
|
||||
to_dict=lambda: {"granularity": "month"}
|
||||
)
|
||||
registry.get_key_pattern.return_value = (
|
||||
"{year}/{month}/{month_name}/{day}/{original_filename}"
|
||||
)
|
||||
request = SimpleNamespace(
|
||||
app=SimpleNamespace(state=SimpleNamespace()),
|
||||
state=SimpleNamespace(),
|
||||
)
|
||||
expected_key = "files/2026/09/Septembre/20/original.pdf"
|
||||
|
||||
with (
|
||||
patch.object(routes, "_storage", return_value=storage),
|
||||
patch.object(routes, "_db", return_value=database),
|
||||
patch.object(routes, "_registry", return_value=registry),
|
||||
patch.object(routes, "get_file", new=AsyncMock(return_value=None)),
|
||||
):
|
||||
asyncio.run(routes.edit_file(
|
||||
request,
|
||||
record.id,
|
||||
DocumentEditRequest(
|
||||
template_name="invoice-template",
|
||||
period_format="monthly",
|
||||
emitting_company="Example Company",
|
||||
document_type="invoice",
|
||||
emission_date=date(2026, 9, 20),
|
||||
),
|
||||
current_user=None,
|
||||
))
|
||||
|
||||
self.assertEqual(record.current_key, expected_key)
|
||||
storage.copy_object.assert_called_once_with(record.original_key, expected_key)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -5,6 +5,7 @@ import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from app.pipeline import DocumentProcessor
|
||||
from app.period import get_period_key_values
|
||||
|
||||
|
||||
class EmissionDateExtractionTests(unittest.TestCase):
|
||||
@@ -41,5 +42,28 @@ class EmissionDateExtractionTests(unittest.TestCase):
|
||||
self.assertIsNotNone(error)
|
||||
|
||||
|
||||
class PeriodKeyPlaceholderTests(unittest.TestCase):
|
||||
def test_key_values_use_the_computed_period_and_representative_day(self):
|
||||
values = get_period_key_values(
|
||||
"2026-01-20",
|
||||
{"granularity": "month", "offset": "previous"},
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
values,
|
||||
{"year": "2025", "month": "12", "month_name": "Décembre", "day": "20"},
|
||||
)
|
||||
|
||||
def test_key_pattern_renders_period_placeholders(self):
|
||||
processor = object.__new__(DocumentProcessor)
|
||||
|
||||
key = processor._format_key_pattern(
|
||||
"{year}/{month}/{month_name}/{day}/document.pdf",
|
||||
**get_period_key_values("2026-09-20", {"granularity": "month"}),
|
||||
)
|
||||
|
||||
self.assertEqual(key, "2026/09/Septembre/20/document.pdf")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -78,4 +78,8 @@ As the owner, I want a standalone Python script that imports my existing paperle
|
||||
- Sidecars now include extracted_text (first page, PyMuPDF, same parameters as the service), so no post-import re-extraction backfill is needed and the corpus is immediately usable for the first retrain.
|
||||
- Optional fields come solely from paperless labels. Labels are free-form in paperless; the template YAML schemas are not enforced at import, and labels colliding with common field names are dropped with a warning.
|
||||
- Document subtype mapping (paperless document types to the renamer template YAML document types) is identity on the name; if the template YAML later diverges, the consistency job surfaces mismatches.
|
||||
- The script lives in the repo as `scripts/import_paperless.py` with its own minimal dependencies (boto3, requests, python-dateutil, PyMuPDF), not in the container image.
|
||||
- The script lives in the repo as `scripts/import_paperless.py` with its own minimal dependencies (boto3, requests, python-dateutil, PyMuPDF), not in the container image.
|
||||
|
||||
## Scope note — dashboard bulk editing
|
||||
|
||||
The initial Paperless-ngx import story remains focused on importing and reconciling the initial corpus. Configurable home-page columns and bulk metadata edits are separate dashboard capabilities and must not change the import's identity, deduplication, or sidecar-source-of-truth rules. Any imported records must expose the same stable metadata and additional-field keys used by the dashboard once indexed.
|
||||
|
||||
@@ -5,8 +5,82 @@ const refreshButton = document.getElementById('refresh-btn');
|
||||
const listStatus = document.getElementById('list-status');
|
||||
const loadMoreButton = document.getElementById('load-more-btn');
|
||||
const loadSentinel = document.getElementById('load-sentinel');
|
||||
const columnFilters = [...document.querySelectorAll('[data-filter]')];
|
||||
const sortButtons = [...document.querySelectorAll('[data-sort]')];
|
||||
let columnFilters = [];
|
||||
const tableHead = document.getElementById('documents-head');
|
||||
const columnOptions = document.getElementById('column-options');
|
||||
const selectedIds = new Set();
|
||||
let allFilteredSelection = false;
|
||||
let availableAdditionalFields = [];
|
||||
const defaultColumns = ['status', 'current_key', 'emitting_company', 'document_type', 'period', 'updated_at'];
|
||||
const columnDefinitions = {
|
||||
status: { label: 'Status', filter: 'status', sortable: true },
|
||||
current_key: { label: 'Document', filter: 'current_key', sortable: true },
|
||||
emitting_company: { label: 'Company', filter: 'emitting_company', sortable: true },
|
||||
document_type: { label: 'Document type', filter: 'document_type', sortable: true },
|
||||
period: { label: 'Period', filter: 'period', sortable: true },
|
||||
updated_at: { label: 'Last modified', filter: 'updated_at', sortable: true },
|
||||
template_name: { label: 'Template', filter: 'template', sortable: true },
|
||||
emission_date: { label: 'Emission date', filter: 'emission_date', sortable: false, inputType: 'date' },
|
||||
confidence: { label: 'Confidence', filter: 'confidence', sortable: true },
|
||||
};
|
||||
let visibleColumns = (() => {
|
||||
try { const saved = JSON.parse(localStorage.getItem('pdf-renamer-columns') || 'null'); return Array.isArray(saved) ? saved : [...defaultColumns]; }
|
||||
catch { return [...defaultColumns]; }
|
||||
})();
|
||||
const selectedCount = () => allFilteredSelection ? total : selectedIds.size;
|
||||
const updateSelectionUi = () => {
|
||||
document.getElementById('selection-count').textContent = allFilteredSelection
|
||||
? `All ${total} filtered documents selected`
|
||||
: `${selectedIds.size} selected`;
|
||||
document.getElementById('bulk-edit-btn').disabled = selectedCount() === 0;
|
||||
const headerSelect = document.getElementById('select-page-head');
|
||||
if (headerSelect) {
|
||||
headerSelect.checked = loadedItems.length > 0 && loadedItems.every((item) => selectedIds.has(item.id));
|
||||
headerSelect.indeterminate = !headerSelect.checked && loadedItems.some((item) => selectedIds.has(item.id));
|
||||
}
|
||||
};
|
||||
const renderColumnPicker = () => {
|
||||
const options = [...Object.entries(columnDefinitions).map(([key, value]) => [key, value.label]),
|
||||
...availableAdditionalFields.map((key) => [`additional:${key}`, key])];
|
||||
columnOptions.innerHTML = options.map(([key, label]) =>
|
||||
`<label><input type="checkbox" data-column-option="${escapeHtml(key)}" ${visibleColumns.includes(key) ? 'checked' : ''}/> ${escapeHtml(label)}</label>`
|
||||
).join('');
|
||||
columnOptions.querySelectorAll('[data-column-option]').forEach((control) => control.addEventListener('change', () => {
|
||||
const key = control.dataset.columnOption;
|
||||
visibleColumns = control.checked ? [...visibleColumns, key] : visibleColumns.filter((item) => item !== key);
|
||||
if (!visibleColumns.length) { visibleColumns = [...defaultColumns]; }
|
||||
localStorage.setItem('pdf-renamer-columns', JSON.stringify(visibleColumns));
|
||||
renderTableHead();
|
||||
updateSortIndicators();
|
||||
renderRows(loadedItems, false);
|
||||
}));
|
||||
};
|
||||
const renderTableHead = () => {
|
||||
const header = (key) => {
|
||||
const def = columnDefinitions[key] || { label: key.replace(/^additional:/, ''), filter: `additional_field__${key.slice(11)}`, sortable: false };
|
||||
return `<th scope="col" ${def.sortable ? `data-sort-column="${key}"` : ''}>${def.sortable ? `<button class="sort-button" data-sort="${key}">${escapeHtml(def.label)} <span></span></button>` : escapeHtml(def.label)}</th>`;
|
||||
};
|
||||
const previousFilterValues = Object.fromEntries(columnFilters.map((filter) => [filter.dataset.filter, filter.value]));
|
||||
const filters = visibleColumns.map((key) => {
|
||||
const def = columnDefinitions[key] || (key.startsWith('additional:') ? { label: key.slice(11), filter: `additional_field__${key.slice(11)}`, inputType: 'search' } : null);
|
||||
if (!def || !def.filter) return '<th></th>';
|
||||
const filterId = `column-filter-${def.filter}`;
|
||||
if (def.filter === 'status') return `<th><label class="visually-hidden" for="${filterId}">Filter status</label><select id="${filterId}" data-filter="status"><option value="">All</option><option value="processed">Processed</option><option value="validated">Validated</option><option value="not processed">Not processed</option><option value="unknown">Unknown</option></select></th>`;
|
||||
const inputType = def.inputType || (def.filter === 'updated_at' ? 'date' : 'search');
|
||||
return `<th><label class="visually-hidden" for="${filterId}">Filter ${escapeHtml(def.label)}</label><input id="${filterId}" data-filter="${def.filter}" type="${inputType}" placeholder="${escapeHtml(def.label)}" value="${escapeHtml(previousFilterValues[def.filter] || '')}" /></th>`;
|
||||
}).join('');
|
||||
tableHead.innerHTML = `<tr><th><input type="checkbox" id="select-page-head" aria-label="Select loaded rows"/></th>${visibleColumns.map(header).join('')}</tr><tr class="column-filters"><th></th>${filters}</tr>`;
|
||||
columnFilters = [...tableHead.querySelectorAll('[data-filter]')];
|
||||
columnFilters.forEach((filter) => filter.addEventListener('input', () => { selectedIds.clear(); allFilteredSelection = false; updateSelectionUi(); loadDocuments(); }));
|
||||
tableHead.querySelectorAll('[data-sort]').forEach((button) => button.addEventListener('click', () => {
|
||||
if (sortBy === button.dataset.sort) sortOrder = sortOrder === 'asc' ? 'desc' : 'asc';
|
||||
else { sortBy = button.dataset.sort; sortOrder = 'asc'; }
|
||||
updateSortIndicators(); loadDocuments();
|
||||
}));
|
||||
const selectHead = tableHead.querySelector('#select-page-head');
|
||||
selectHead.addEventListener('change', () => selectLoadedRows(selectHead.checked));
|
||||
};
|
||||
let sortButtons = [];
|
||||
let sortBy = 'updated_at';
|
||||
let sortOrder = 'desc';
|
||||
let page = 1;
|
||||
@@ -76,37 +150,36 @@ const renderSummary = (totals) => {
|
||||
|
||||
const renderRows = (items, append) => {
|
||||
if (!items.length && !append) {
|
||||
body.innerHTML = '<tr><td colspan="8" class="empty-state">No documents found.</td></tr>';
|
||||
body.innerHTML = `<tr><td colspan="${visibleColumns.length + 1}" class="empty-state">No documents found.</td></tr>`;
|
||||
return;
|
||||
}
|
||||
|
||||
const listParams = getListParams();
|
||||
const rows = items
|
||||
.map((item) => {
|
||||
const detailParams = new URLSearchParams(listParams);
|
||||
detailParams.set('id', item.id);
|
||||
return `
|
||||
<tr>
|
||||
<td><span class="badge ${escapeHtml(badgeClass(item.status))}">${escapeHtml(item.status || 'unknown')}</span></td>
|
||||
<td>
|
||||
<a class="document-link" href="/document.html?${detailParams.toString()}">${escapeHtml(item.current_key?.split('/').pop() || 'Untitled document')}</a>
|
||||
<div class="key-text">${escapeHtml(item.current_key || '—')}</div>
|
||||
</td>
|
||||
<td>${escapeHtml(item.template_name || '—')}</td>
|
||||
<td>${escapeHtml(item.emitting_company || '—')}</td>
|
||||
<td>${escapeHtml(item.document_type || '—')}</td>
|
||||
<td>${escapeHtml(item.period || '—')}</td>
|
||||
<td>${item.confidence == null ? '—' : `${(Number(item.confidence) * 100).toFixed(1)}%`}</td>
|
||||
<td>${escapeHtml(formatUpdated(item.updated_at))}</td>
|
||||
</tr>
|
||||
`;
|
||||
})
|
||||
.join('');
|
||||
if (append) {
|
||||
body.insertAdjacentHTML('beforeend', rows);
|
||||
} else {
|
||||
body.innerHTML = rows;
|
||||
}
|
||||
const rows = items.map((item) => {
|
||||
const detailParams = new URLSearchParams(listParams);
|
||||
detailParams.set('id', item.id);
|
||||
const cells = visibleColumns.map((key) => {
|
||||
if (key.startsWith('additional:')) return `<td>${escapeHtml(item.optional_fields?.[key.slice(11)] ?? '—')}</td>`;
|
||||
if (key === 'status') return `<td><span class="badge ${escapeHtml(badgeClass(item.status))}">${escapeHtml(item.status || 'unknown')}</span></td>`;
|
||||
if (key === 'current_key') return `<td><a class="document-link" href="/document.html?${detailParams.toString()}">${escapeHtml(item.current_key?.split('/').pop() || 'Untitled document')}</a><div class="key-text">${escapeHtml(item.current_key || '—')}</div></td>`;
|
||||
if (key === 'updated_at') return `<td>${escapeHtml(formatUpdated(item.updated_at))}</td>`;
|
||||
if (key === 'confidence') return `<td>${item.confidence == null ? '—' : `${(Number(item.confidence) * 100).toFixed(1)}%`}</td>`;
|
||||
return `<td>${escapeHtml(item[key] ?? '—')}</td>`;
|
||||
}).join('');
|
||||
return `<tr><td><input type="checkbox" data-select-id="${escapeHtml(item.id)}" aria-label="Select document" ${selectedIds.has(item.id) ? 'checked' : ''}/></td>${cells}</tr>`;
|
||||
}).join('');
|
||||
if (append) body.insertAdjacentHTML('beforeend', rows); else body.innerHTML = rows;
|
||||
body.querySelectorAll('[data-select-id]').forEach((control) => control.addEventListener('change', () => {
|
||||
if (control.checked) selectedIds.add(control.dataset.selectId); else selectedIds.delete(control.dataset.selectId);
|
||||
allFilteredSelection = false; updateSelectionUi();
|
||||
}));
|
||||
updateSelectionUi();
|
||||
};
|
||||
const selectLoadedRows = (checked) => {
|
||||
if (checked) loadedItems.forEach((item) => selectedIds.add(item.id));
|
||||
else loadedItems.forEach((item) => selectedIds.delete(item.id));
|
||||
allFilteredSelection = false;
|
||||
renderRows(loadedItems, false);
|
||||
updateSelectionUi();
|
||||
};
|
||||
|
||||
const loadDocuments = async ({ append = false } = {}) => {
|
||||
@@ -147,6 +220,8 @@ const loadDocuments = async ({ append = false } = {}) => {
|
||||
});
|
||||
renderRows(items, append);
|
||||
loadedItems = append ? loadedItems.concat(items) : items;
|
||||
availableAdditionalFields = [...new Set(loadedItems.flatMap((item) => Object.keys(item.optional_fields || {})))];
|
||||
renderColumnPicker();
|
||||
page += 1;
|
||||
hasMore = loadedItems.length < total;
|
||||
listStatus.textContent = total
|
||||
@@ -173,6 +248,7 @@ const loadDocuments = async ({ append = false } = {}) => {
|
||||
};
|
||||
|
||||
const updateSortIndicators = () => {
|
||||
sortButtons = [...document.querySelectorAll('[data-sort]')];
|
||||
document.querySelectorAll('[data-sort-column]').forEach((header) => {
|
||||
const active = header.dataset.sortColumn === sortBy;
|
||||
header.setAttribute('aria-sort', active ? (sortOrder === 'asc' ? 'ascending' : 'descending') : 'none');
|
||||
@@ -181,8 +257,17 @@ const updateSortIndicators = () => {
|
||||
});
|
||||
};
|
||||
|
||||
columnFilters.forEach((filter) => filter.addEventListener('input', loadDocuments));
|
||||
searchFilter.addEventListener('input', loadDocuments);
|
||||
renderColumnPicker();
|
||||
renderTableHead();
|
||||
fetch('/document-options').then((response) => response.ok ? response.json() : null).then((options) => {
|
||||
if (options && Array.isArray(options.optional_field_names)) {
|
||||
availableAdditionalFields = [...new Set([...availableAdditionalFields, ...options.optional_field_names])];
|
||||
renderColumnPicker();
|
||||
}
|
||||
}).catch(() => {});
|
||||
searchFilter.addEventListener('input', () => {
|
||||
selectedIds.clear(); allFilteredSelection = false; updateSelectionUi(); loadDocuments();
|
||||
});
|
||||
refreshButton.addEventListener('click', loadDocuments);
|
||||
loadMoreButton.addEventListener('click', () => loadDocuments({ append: true }));
|
||||
sortButtons.forEach((button) => {
|
||||
@@ -198,6 +283,131 @@ sortButtons.forEach((button) => {
|
||||
});
|
||||
});
|
||||
|
||||
document.getElementById('select-filtered-btn').addEventListener('click', () => {
|
||||
allFilteredSelection = true;
|
||||
selectedIds.clear();
|
||||
updateSelectionUi();
|
||||
});
|
||||
document.getElementById('clear-selection-btn').addEventListener('click', () => {
|
||||
selectedIds.clear(); allFilteredSelection = false; renderRows(loadedItems, false); updateSelectionUi();
|
||||
});
|
||||
const bulkOptions = { names: [], values: {} };
|
||||
const setBulkSelectOptions = (select, values, placeholder = 'Keep unchanged') => {
|
||||
select.replaceChildren();
|
||||
const empty = document.createElement('option');
|
||||
empty.value = '';
|
||||
empty.textContent = placeholder;
|
||||
select.append(empty);
|
||||
[...new Set((values || []).filter((value) => value !== null && value !== undefined && String(value).trim() !== ''))].sort((a,b) => String(a).localeCompare(String(b))).forEach((value) => {
|
||||
const option = document.createElement('option');
|
||||
option.value = value;
|
||||
option.textContent = value;
|
||||
select.append(option);
|
||||
});
|
||||
};
|
||||
const renderBulkAdditionalRow = (fieldName = '') => {
|
||||
const host = document.getElementById('bulk-additional-fields');
|
||||
const row = document.createElement('div');
|
||||
row.className = 'additional-field-row bulk-additional-row';
|
||||
const nameGroup = document.createElement('div');
|
||||
nameGroup.className = 'field-group';
|
||||
const nameLabel = document.createElement('label');
|
||||
nameLabel.textContent = 'Additional field';
|
||||
const name = document.createElement('select');
|
||||
name.className = 'bulk-additional-name';
|
||||
name.setAttribute('aria-label', 'Additional field');
|
||||
setBulkSelectOptions(name, bulkOptions.names, 'Choose field');
|
||||
if (fieldName && ![...name.options].some((option) => option.value === fieldName)) {
|
||||
const option = document.createElement('option'); option.value = fieldName; option.textContent = fieldName; name.append(option);
|
||||
}
|
||||
name.value = fieldName;
|
||||
nameGroup.append(nameLabel, name);
|
||||
|
||||
const valueGroup = document.createElement('div');
|
||||
valueGroup.className = 'field-group';
|
||||
const valueLabel = document.createElement('label');
|
||||
valueLabel.textContent = 'Value';
|
||||
const value = document.createElement('select');
|
||||
value.className = 'bulk-additional-value';
|
||||
value.setAttribute('aria-label', 'Additional field value');
|
||||
const populateValues = () => {
|
||||
setBulkSelectOptions(value, bulkOptions.values[name.value] || [], 'Choose value');
|
||||
const clear = document.createElement('option');
|
||||
clear.value = '__CLEAR_FIELD__';
|
||||
clear.textContent = 'Clear field';
|
||||
value.append(clear);
|
||||
};
|
||||
populateValues();
|
||||
name.addEventListener('change', populateValues);
|
||||
valueGroup.append(valueLabel, value);
|
||||
|
||||
const remove = document.createElement('button');
|
||||
remove.type = 'button'; remove.className = 'secondary-btn small-btn'; remove.textContent = 'Remove';
|
||||
remove.addEventListener('click', () => row.remove());
|
||||
row.append(nameGroup, valueGroup, remove);
|
||||
host.append(row);
|
||||
};
|
||||
fetch('/document-options').then((response) => response.ok ? response.json() : null).then((options) => {
|
||||
if (!options) return;
|
||||
bulkOptions.names = options.optional_field_names || [];
|
||||
bulkOptions.values = options.optional_field_values || {};
|
||||
setBulkSelectOptions(document.getElementById('bulk-template'), options.templates || []);
|
||||
setBulkSelectOptions(document.getElementById('bulk-period-format'), options.period_formats || []);
|
||||
setBulkSelectOptions(document.getElementById('bulk-company'), options.companies || []);
|
||||
setBulkSelectOptions(document.getElementById('bulk-type'), options.document_types || []);
|
||||
}).catch((error) => console.error('Unable to load bulk edit options', error));
|
||||
document.getElementById('bulk-add-field').addEventListener('click', () => renderBulkAdditionalRow());
|
||||
document.getElementById('bulk-edit-btn').addEventListener('click', () => {
|
||||
document.getElementById('bulk-edit-form').hidden = false;
|
||||
document.getElementById('bulk-scope').textContent = allFilteredSelection
|
||||
? `This will apply to all ${total} documents matching the active filters.`
|
||||
: `This will apply to ${selectedIds.size} selected documents.`;
|
||||
document.getElementById('bulk-result').textContent = '';
|
||||
});
|
||||
document.getElementById('cancel-bulk-edit').addEventListener('click', () => { document.getElementById('bulk-edit-form').hidden = true; });
|
||||
document.getElementById('bulk-edit-form').addEventListener('submit', async (event) => {
|
||||
event.preventDefault();
|
||||
const changes = {};
|
||||
for (const [id, key] of [['bulk-template', 'template_name'], ['bulk-period-format', 'period_format'], ['bulk-company', 'emitting_company'], ['bulk-type', 'document_type']]) {
|
||||
const value = document.getElementById(id).value;
|
||||
if (value) changes[key] = value;
|
||||
}
|
||||
const emissionDate = document.getElementById('bulk-emission-date').value;
|
||||
if (emissionDate) changes.emission_date = emissionDate;
|
||||
const additionalFields = {};
|
||||
for (const row of document.querySelectorAll('.bulk-additional-row')) {
|
||||
const name = row.querySelector('.bulk-additional-name').value;
|
||||
const value = row.querySelector('.bulk-additional-value').value;
|
||||
if (!name || !value) continue;
|
||||
if (Object.prototype.hasOwnProperty.call(additionalFields, name)) {
|
||||
document.getElementById('bulk-result').textContent = `Choose each additional field only once: ${name}`;
|
||||
return;
|
||||
}
|
||||
additionalFields[name] = value === '__CLEAR_FIELD__' ? null : value;
|
||||
}
|
||||
if (Object.keys(additionalFields).length) changes.additional_fields = additionalFields;
|
||||
if (!Object.keys(changes).length) {
|
||||
document.getElementById('bulk-result').textContent = 'Choose at least one field to change.';
|
||||
return;
|
||||
}
|
||||
const payload = allFilteredSelection
|
||||
? { all_matching: true, filters: Object.fromEntries(getListParams()), changes }
|
||||
: { ids: [...selectedIds], changes };
|
||||
const result = document.getElementById('bulk-result');
|
||||
const summary = Object.entries(changes).map(([key, value]) => key === 'additional_fields'
|
||||
? `Additional fields: ${Object.entries(value).map(([field, fieldValue]) => `${field} → ${fieldValue === null ? 'clear' : fieldValue}`).join(', ')}`
|
||||
: `${key}: ${value}`).join('; ');
|
||||
if (!window.confirm(`Apply to ${allFilteredSelection ? total : selectedIds.size} document(s)?\n\n${summary}`)) return;
|
||||
result.textContent = 'Applying changes…';
|
||||
try {
|
||||
const response = await fetch('/files/bulk-edit', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) });
|
||||
const data = await response.json();
|
||||
if (!response.ok) throw new Error(data.detail || `HTTP ${response.status}`);
|
||||
result.textContent = `${data.succeeded} succeeded, ${data.failed} failed out of ${data.requested}.`;
|
||||
selectedIds.clear(); allFilteredSelection = false;
|
||||
await loadDocuments();
|
||||
} catch (error) { result.textContent = `Bulk edit failed: ${error.message}`; }
|
||||
});
|
||||
updateSortIndicators();
|
||||
loadDocuments();
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ saveButton.disabled = true;
|
||||
let currentFileId = null;
|
||||
let fieldSuggestionState = { names: [], values: {} };
|
||||
let initialDocumentState = null;
|
||||
let isSaving = false;
|
||||
|
||||
const canonicalizeState = (value) => {
|
||||
if (value === null || value === undefined) return '';
|
||||
@@ -53,13 +54,7 @@ const getCurrentDocumentState = () => ({
|
||||
});
|
||||
|
||||
const updateSaveButtonState = () => {
|
||||
if (!currentFileId || !initialDocumentState) {
|
||||
saveButton.disabled = true;
|
||||
return;
|
||||
}
|
||||
const currentState = canonicalizeState(getCurrentDocumentState());
|
||||
const baselineState = canonicalizeState(initialDocumentState);
|
||||
saveButton.disabled = JSON.stringify(currentState) === JSON.stringify(baselineState);
|
||||
saveButton.disabled = !currentFileId || !initialDocumentState || isSaving;
|
||||
};
|
||||
|
||||
const openNeighbor = (neighborId) => {
|
||||
@@ -576,7 +571,8 @@ const saveDocument = async (event) => {
|
||||
emissionDateInput.setCustomValidity(emissionDate ? '' : 'Enter a valid date in dd/mm/yyyy format.');
|
||||
if (!editForm.reportValidity()) return;
|
||||
|
||||
saveButton.disabled = true;
|
||||
isSaving = true;
|
||||
updateSaveButtonState();
|
||||
editStatus.textContent = 'Recalculating document name and saving...';
|
||||
const payload = {
|
||||
template_name: templateSelect.value,
|
||||
@@ -603,7 +599,8 @@ const saveDocument = async (event) => {
|
||||
editStatus.textContent = `Unable to save changes: ${error.message}`;
|
||||
console.error(error);
|
||||
} finally {
|
||||
saveButton.disabled = false;
|
||||
isSaving = false;
|
||||
updateSaveButtonState();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -40,33 +40,39 @@
|
||||
</section>
|
||||
|
||||
<section class="table-card">
|
||||
<div class="list-tools">
|
||||
<details id="column-picker"><summary>Choose columns</summary><div id="column-options" class="column-options"></div></details>
|
||||
<button id="select-filtered-btn" class="secondary-btn" type="button">Select all filtered</button>
|
||||
<button id="clear-selection-btn" class="secondary-btn" type="button">Clear selection</button>
|
||||
<span id="selection-count" role="status">0 selected</span>
|
||||
<button id="bulk-edit-btn" class="primary-btn" type="button" disabled>Bulk edit</button>
|
||||
</div>
|
||||
<form id="bulk-edit-form" class="bulk-edit-form" hidden>
|
||||
<h3>Bulk edit selected documents</h3>
|
||||
<p id="bulk-scope"></p>
|
||||
<p class="config-description">Only fields you change below will be updated. Other metadata remains unchanged.</p>
|
||||
<div class="bulk-edit-grid">
|
||||
<div class="field-group"><label for="bulk-template">Template</label><select id="bulk-template"><option value="">Keep unchanged</option></select></div>
|
||||
<div class="field-group"><label for="bulk-period-format">Period format</label><select id="bulk-period-format"><option value="">Keep unchanged</option></select></div>
|
||||
<div class="field-group"><label for="bulk-company">Company</label><select id="bulk-company"><option value="">Keep unchanged</option></select></div>
|
||||
<div class="field-group"><label for="bulk-type">Document type</label><select id="bulk-type"><option value="">Keep unchanged</option></select></div>
|
||||
<div class="field-group"><label for="bulk-emission-date">Emission date</label><input id="bulk-emission-date" type="date" /></div>
|
||||
</div>
|
||||
<div class="additional-data">
|
||||
<div class="additional-header"><h4>Additional fields</h4><button id="bulk-add-field" class="secondary-btn" type="button">Add field</button></div>
|
||||
<p class="field-hint">Choose a field and a value, or choose “Clear field” to remove it from selected documents.</p>
|
||||
<div id="bulk-additional-fields" class="additional-field-list"></div>
|
||||
</div>
|
||||
<div class="edit-form-actions">
|
||||
<button class="primary-btn" type="submit">Review and apply changes</button>
|
||||
<button id="cancel-bulk-edit" class="secondary-btn" type="button">Cancel</button>
|
||||
<p id="bulk-result" role="status" aria-live="polite"></p>
|
||||
</div>
|
||||
</form>
|
||||
<table class="documents-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th scope="col" data-sort-column="status"><button class="sort-button" data-sort="status">Status <span></span></button></th>
|
||||
<th scope="col" data-sort-column="current_key"><button class="sort-button" data-sort="current_key">Document <span></span></button></th>
|
||||
<th scope="col" data-sort-column="template_name"><button class="sort-button" data-sort="template_name">Template <span></span></button></th>
|
||||
<th scope="col" data-sort-column="emitting_company"><button class="sort-button" data-sort="emitting_company">Company <span></span></button></th>
|
||||
<th scope="col" data-sort-column="document_type"><button class="sort-button" data-sort="document_type">Type <span></span></button></th>
|
||||
<th scope="col" data-sort-column="period"><button class="sort-button" data-sort="period">Period <span></span></button></th>
|
||||
<th scope="col" data-sort-column="confidence"><button class="sort-button" data-sort="confidence">Confidence <span></span></button></th>
|
||||
<th scope="col" data-sort-column="updated_at" aria-sort="descending"><button class="sort-button" data-sort="updated_at">Updated <span>↓</span></button></th>
|
||||
</tr>
|
||||
<tr class="column-filters">
|
||||
<th><label class="visually-hidden" for="filter-status">Filter by status</label><select id="filter-status" data-filter="status"><option value="">All</option><option value="processed">Processed</option><option value="validated">Validated</option><option value="not processed">Not processed</option><option value="unknown">Unknown</option></select></th>
|
||||
<th><label class="visually-hidden" for="filter-document">Filter by document</label><input id="filter-document" data-filter="current_key" type="search" placeholder="Name or key" /></th>
|
||||
<th><label class="visually-hidden" for="filter-template">Filter by template</label><input id="filter-template" data-filter="template" type="search" placeholder="Template" /></th>
|
||||
<th><label class="visually-hidden" for="filter-company">Filter by company</label><input id="filter-company" data-filter="emitting_company" type="search" placeholder="Company" /></th>
|
||||
<th><label class="visually-hidden" for="filter-type">Filter by type</label><input id="filter-type" data-filter="document_type" type="search" placeholder="Type" /></th>
|
||||
<th><label class="visually-hidden" for="filter-period">Filter by period</label><input id="filter-period" data-filter="period" type="search" placeholder="Period" /></th>
|
||||
<th><label class="visually-hidden" for="filter-confidence">Filter by confidence</label><input id="filter-confidence" data-filter="confidence" type="search" placeholder="Percent" /></th>
|
||||
<th><label class="visually-hidden" for="filter-updated">Filter by updated date</label><input id="filter-updated" data-filter="updated_at" type="date" /></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<thead id="documents-head"></thead>
|
||||
<tbody id="documents-body">
|
||||
<tr>
|
||||
<td colspan="8" class="empty-state">Loading documents...</td>
|
||||
</tr>
|
||||
<tr><td colspan="8" class="empty-state">Loading documents...</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
|
||||
@@ -921,3 +921,62 @@ body {
|
||||
outline: 2px solid currentColor;
|
||||
outline-offset: 2px;
|
||||
}
|
||||
|
||||
|
||||
/* Configurable document list and bulk-edit controls */
|
||||
.list-tools {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
gap: 0.75rem;
|
||||
padding: 1rem;
|
||||
border-bottom: 1px solid var(--border, #d9dee8);
|
||||
}
|
||||
.column-options {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(9rem, 1fr));
|
||||
gap: 0.5rem;
|
||||
min-width: min(32rem, 80vw);
|
||||
padding: 0.75rem;
|
||||
}
|
||||
.column-options label,
|
||||
.list-tools > label {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 0.4rem;
|
||||
}
|
||||
.bulk-edit-form {
|
||||
display: grid;
|
||||
gap: 0.8rem;
|
||||
padding: 1rem;
|
||||
border-bottom: 1px solid var(--border, #d9dee8);
|
||||
}
|
||||
.bulk-edit-form[hidden] { display: none; }
|
||||
.bulk-edit-form .field-group { max-width: 36rem; }
|
||||
.bulk-edit-form textarea { width: 100%; }
|
||||
#selection-count { font-weight: 600; }
|
||||
|
||||
|
||||
.bulk-edit-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(220px, 1fr));
|
||||
gap: 1rem;
|
||||
}
|
||||
.bulk-additional-row {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(180px, 1fr) minmax(180px, 1fr) auto;
|
||||
align-items: end;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
.bulk-additional-row .field-group { min-width: 0; }
|
||||
.bulk-edit-form .additional-data { margin-top: 0.5rem; }
|
||||
.bulk-edit-form .edit-form-actions {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
.bulk-edit-form .edit-form-actions .action-status { margin: 0; }
|
||||
@media (max-width: 680px) {
|
||||
.bulk-additional-row { grid-template-columns: 1fr; }
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user