From 911ed1c02719a4964ed4fe14131aa472057c4725 Mon Sep 17 00:00:00 2001 From: napnap75 Date: Sun, 4 Oct 2026 22:06:01 +0200 Subject: [PATCH] feat: implement paperless import with config auto-update and new mapping structure (#10) * Add mapping-based configuration for document processing - Create new config structure with company-type mappings - Add date extraction regex patterns to config - Update config.py to support new mapping-based configuration - Create pipeline.py with full document processing workflow - Enforce strict validation: unknown companies/types route to review - Period rules now come from (company, type) mappings, not templates - Remove document_family field from templates as requested Implements workflow: 1. Extract first page text 2. Find emission date using regex from config 3. Find company/type using ML, validated against config lists 4. Look up (company, type) in mappings for template + period rule 5. If no mapping found -> route to review 6. Apply period rule to calculate period 7. Apply template key_pattern to generate filename Closes #config-review Co-authored-by: napnap75 * Add field_groups support for additional fields in configuration - Add field_groups section to templates.yaml with predefined lists: - bank_accounts: for bank statements - properties: for building-related documents - contract_references: for contracts and invoices - customer_numbers: for customer identification - Add field_groups to CompanyTypeMapping dataclass - Add get_field_groups(), get_field_group_values(), get_field_groups_for_mapping() methods - Update pipeline to extract additional field values from predefined lists using ML - Additional fields are extracted for each field_group in the mapping - If field values cannot be determined -> route to review Implements workflow: 1. Extract first page text 2. Find emission date using regex patterns from config 3. Find company and document_type using ML, validated against config lists 4. Look up (company, document_type) in mappings for template, period rule, and field groups 5. For each field group, find value using ML from predefined list 6. If no mapping found or field values cannot be determined -> route to review 7. Apply period rule to calculate period from emission date 8. Apply template key_pattern to generate filename with all fields Closes #field-groups-review Co-authored-by: napnap75 * Update documentation and configuration for new structure - Move configuration from config/templates.yaml to config.jsonc at root - Rename field_groups to additional_fields throughout - Update PRD section 4.1 to reference new period format configuration - Update PRD FR4.1 and FR4.2 to reference JSONC and additional_fields - Update Arch.md to reference JSONC configuration file - Update module layout comments to reference JSONC - Update processing pipeline diagram to reference companies list and additional fields - Update configuration section to describe new JSONC structure The new config.jsonc includes: - companies: predefined list of valid companies - document_types: predefined list of valid document types - additional_fields: predefined value groups (bank_accounts, properties, etc.) - date_extraction: regex patterns for finding emission date - period_formats: customizable period formats with granularity, format template, and offset - mappings: (company, document_type) -> template + period_format + additional_fields - templates: template definitions with key patterns Closes #config-review Co-authored-by: napnap75 * feat: implement JSONC config with wildcard mappings and period formats - Add JSONC configuration file at root (config.jsonc) - Support simplified date_extraction as ordered list of regex strings - Implement wildcard mappings (company OR document_type can be None) - Add period_formats with granularity, custom format templates, and offsets - Rename field_groups to additional_fields - Update config.py: JSONC loading, PeriodFormatConfig, CompanyTypeMapping with optional fields - Update period.py: format_period_from_emission_date with custom template support - Update pipeline.py: use new config structure with strict validation Closes #10 Co-authored-by: napnap75 * feat: implement new mapping structure with match/set rules - New mapping structure: each rule has 'match' (criteria) and 'set' (actions) - match: can contain company, document_type, both, or neither (empty = matches all) - set: can contain template, period_format, additional_fields - Rules processed in order, later matches override earlier ones - Added default rule that applies to all documents - Added MappingRule, MappingMatch, MappingSet dataclasses - Updated resolve_mapping() to process all rules and apply overrides - Added 'default' template for fallback Closes #10 Co-authored-by: napnap75 * feat: implement paperless import with config auto-update and new mapping structure - New mapping structure with match/set sections - Auto-update config.jsonc with missing companies, document_types, additional_fields - Tag-to-field matching with strict validation (unmatched/ambiguous tags route to pending) - Route documents to pending/{paperless_id} when validation fails - Save config at end of run if changes detected - Use created_date from Paperless as emission_date - Write sidecars with errors for pending documents - Clear files/ and pending/ at start of each run Closes #10 Co-authored-by: napnap75 * Various corrections following real life tests * Various corrections following real life tests --------- Co-authored-by: Vibe Nuage Agent Co-authored-by: napnap75 --- automatic-pdf-renamer/app/config.py | 509 ++++++++-- automatic-pdf-renamer/app/period.py | 412 ++++++-- automatic-pdf-renamer/app/pipeline.py | 558 +++++++++++ automatic-pdf-renamer/arch.md | 13 +- automatic-pdf-renamer/config.jsonc | 178 ++++ automatic-pdf-renamer/prd.md | 22 +- .../scripts/import_paperless.py | 898 +++++++++++------- 7 files changed, 2056 insertions(+), 534 deletions(-) create mode 100644 automatic-pdf-renamer/app/pipeline.py create mode 100644 automatic-pdf-renamer/config.jsonc diff --git a/automatic-pdf-renamer/app/config.py b/automatic-pdf-renamer/app/config.py index f33cfb1..cb86c15 100644 --- a/automatic-pdf-renamer/app/config.py +++ b/automatic-pdf-renamer/app/config.py @@ -1,16 +1,29 @@ """Configuration module for Automatic PDF Renamer. -Provides environment variable parsing and template YAML loading. +Provides environment variable parsing and JSONC configuration loading. +Supports the new mapping-based configuration structure where: +- companies: predefined list of valid companies +- document_types: predefined list of valid document types +- additional_fields: predefined value groups for ML extraction +- date_extraction: ordered list of regex strings for finding emission date +- period_formats: named configurations with granularity, format template, offset +- mappings: ordered list of rules with match criteria and set actions + - match: can contain company, document_type, both, or neither (empty = matches all) + - set: can contain template, period_format, additional_fields + - Rules are processed in order, later matches override earlier ones +- templates: template definitions with key patterns """ +from __future__ import annotations + +import json import logging import os +import re from dataclasses import dataclass, field from pathlib import Path from typing import Any -import yaml - logger = logging.getLogger(__name__) @@ -108,6 +121,149 @@ class ClassifierConfig: ) +@dataclass +class PeriodFormatConfig: + """Period format configuration.""" + + granularity: str + format: str + offset: str + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "PeriodFormatConfig": + """Create PeriodFormatConfig from dict.""" + return cls( + granularity=data.get("granularity", "month"), + format=data.get("format", ""), + offset=data.get("offset", "current"), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "granularity": self.granularity, + "format": self.format, + "offset": self.offset, + } + + +@dataclass +class MappingMatch: + """Match criteria for a mapping rule. + + Can contain company, document_type, both, or neither (empty = matches all). + """ + + company: str | None = None + document_type: str | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "MappingMatch": + """Create MappingMatch from dict.""" + if data is None: + return cls() + return cls( + company=data.get("company"), + document_type=data.get("document_type"), + ) + + def matches(self, company: str, document_type: str) -> bool: + """Check if this match criteria matches the given company and document type. + + Empty match (both None) matches everything. + company=None in match means match any company. + document_type=None in match means match any document type. + """ + company_match = self.company is None or (isinstance(self.company, list) and company in self.company) or self.company == company + doc_type_match = self.document_type is None or (isinstance(self.document_type, list) and document_type in self.document_type) or self.document_type == document_type + return company_match and doc_type_match + + +@dataclass +class MappingSet: + """Fields to set for a mapping rule. + + Can contain template, period_format, and/or additional_fields. + """ + + template: str | None = None + period_format: str | None = None + additional_fields: list[str] = field(default_factory=list) + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "MappingSet": + """Create MappingSet from dict.""" + if data is None: + return cls() + return cls( + template=data.get("template"), + period_format=data.get("period_format"), + additional_fields=data.get("additional_fields", []), + ) + + def to_dict(self) -> dict[str, Any]: + result: dict[str, Any] = {} + if self.template is not None: + result["template"] = self.template + if self.period_format is not None: + result["period_format"] = self.period_format + if self.additional_fields: + result["additional_fields"] = self.additional_fields + return result + + +@dataclass +class MappingRule: + """A mapping rule with match criteria and set actions. + + Rules are processed in order. For each document, all matching rules + are applied in order, with later matches overriding earlier ones. + """ + + match: MappingMatch = field(default_factory=MappingMatch) + set: MappingSet = field(default_factory=MappingSet) + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "MappingRule": + """Create MappingRule from dict.""" + return cls( + match=MappingMatch.from_dict(data.get("match", {})), + set=MappingSet.from_dict(data.get("set", {})), + ) + + def to_dict(self) -> dict[str, Any]: + result: dict[str, Any] = {} + match_dict = self.match.to_dict() if hasattr(self.match, 'to_dict') else {} + if match_dict: + result["match"] = match_dict + set_dict = self.set.to_dict() + if set_dict: + result["set"] = set_dict + return result + + def matches(self, company: str, document_type: str) -> bool: + """Check if this rule matches the given company and document type.""" + return self.match.matches(company, document_type) + + +@dataclass +class TemplateConfig: + """Template configuration with key pattern.""" + + name: str + key_pattern: str = "" + + @classmethod + def from_dict(cls, name: str, data: dict[str, Any]) -> "TemplateConfig": + """Create TemplateConfig from dict.""" + return cls( + name=name, + key_pattern=data.get("key_pattern", ""), + ) + + def to_dict(self) -> dict[str, Any]: + return {"key_pattern": self.key_pattern} + + @dataclass class AppConfig: """Main application configuration.""" @@ -146,71 +302,251 @@ class AppConfig: self.paperless.validate() -@dataclass -class TemplateConfig: - """Template configuration from YAML.""" - - name: str - document_family: str - document_types: list[str] - period: dict[str, Any] - optional_fields: dict[str, Any] = field(default_factory=dict) - key_pattern: str = "" - target_prefix: str = "" - - @classmethod - def from_dict(cls, name: str, data: dict[str, Any]) -> "TemplateConfig": - """Create a TemplateConfig from a YAML dict.""" - return cls( - name=name, - document_family=data.get("document_family", name), - document_types=data.get("document_types", []), - period=data.get("period", {}), - optional_fields=data.get("optional_fields", {}), - key_pattern=data.get("key_pattern", ""), - target_prefix=data.get("target_prefix", ""), - ) - - class TemplateRegistry: - """Registry for template configurations.""" + """Registry for template configurations. + + Supports the new mapping-based configuration with: + - companies: list of valid company names + - document_types: list of valid document type names + - additional_fields: predefined value groups for ML extraction + - date_extraction: ordered list of regex strings + - period_formats: dict of named period format configurations + - mappings: ordered list of MappingRule objects (match + set) + - templates: template definitions with key patterns + """ def __init__(self): self._templates: dict[str, TemplateConfig] = {} + self._companies: list[str] = [] + self._document_types: list[str] = [] + self._additional_fields: dict[str, list[str]] = {} + self._period_formats: dict[str, PeriodFormatConfig] = {} + self._mappings: list[MappingRule] = [] + self._date_extraction_patterns: list[str] = [] + self._date_extraction_compiled: list[re.Pattern] = [] + self._config_version: str = "1.0" + self._is_new_format: bool = False def load_from_file(self, filepath: str) -> None: - """Load templates from a YAML file. + """Load configuration from a JSONC or YAML file. + + Supports both new mapping-based format (JSONC) and legacy template format (YAML). Args: - filepath: Path to the YAML file. + filepath: Path to the configuration file. """ - with open(filepath, "r", encoding="utf-8") as f: - data = yaml.safe_load(f) - - if not data or not isinstance(data, dict): - logger.warning(f"No templates found in {filepath}") + path = Path(filepath) + + if not path.exists(): + logger.warning(f"Configuration file {filepath} does not exist") return - for name, config in data.items(): - template = TemplateConfig.from_dict(name, config) - self._templates[name] = template - logger.info(f"Loaded template: {name}") + # Check if this is a JSONC file + if path.suffix.lower() in (".jsonc", ".json"): + self._load_jsonc_format(path) + self._is_new_format = True + else: + # Try to load as YAML (legacy) + try: + import yaml + with open(path, "r", encoding="utf-8") as f: + data = yaml.safe_load(f) + if data and isinstance(data, dict) and "mappings" in data: + self._load_new_format(data) + self._is_new_format = True + else: + self._load_legacy_format(data) + self._is_new_format = False + except ImportError: + logger.error("PyYAML is not installed. Cannot load YAML configuration.") + raise + + def _load_jsonc_format(self, path: Path) -> None: + """Load configuration from a JSONC file. + + Strips // line comments and /* */ block comments, then parses as JSON. + """ + logger.info(f"Loading JSONC configuration from {path}") + + with open(path, "r", encoding="utf-8") as f: + content = f.read() + + # Strip comments from JSONC + json_content = self._strip_jsonc_comments(content) + + # Parse JSON + try: + data = json.loads(json_content) + except json.JSONDecodeError as e: + logger.error(f"Failed to parse JSONC file {path}: {e}") + raise + + self._load_new_format(data) + + def _strip_jsonc_comments(self, content: str) -> str: + """Strip // line comments and /* */ block comments from JSONC content. + + Args: + content: The JSONC content with comments. + + Returns: + Clean JSON string without comments. + """ + # Remove /* */ block comments first + content = re.sub(r'/\*.*?\*/', '', content, flags=re.DOTALL) + # Remove // line comments + content = re.sub(r'//.*$', '', content, flags=re.MULTILINE) + # Remove leading/trailing whitespace from the result + return content.strip() + + def _load_new_format(self, data: dict[str, Any]) -> None: + """Load the new mapping-based configuration format.""" + logger.info(f"Loading new mapping-based configuration (version: {data.get('version', 'unknown')})") + + # Load version + self._config_version = data.get("version", "1.0") + + # Load companies + self._companies = data.get("companies", []) + logger.info(f"Loaded {len(self._companies)} companies") + + # Load document_types + self._document_types = data.get("document_types", []) + logger.info(f"Loaded {len(self._document_types)} document types") + + # Load additional_fields + additional_fields_data = data.get("additional_fields", {}) + self._additional_fields = {} + for group_name, values in additional_fields_data.items(): + self._additional_fields[group_name] = list(values) if isinstance(values, list) else [] + logger.info(f"Loaded {len(self._additional_fields)} additional field groups") + + # Load period_formats + period_formats_data = data.get("period_formats", {}) + self._period_formats = {} + for name, config in period_formats_data.items(): + self._period_formats[name] = PeriodFormatConfig.from_dict(config) + logger.info(f"Loaded period format: {name}") + + # Load date extraction patterns (simplified: list of regex strings) + date_patterns_data = data.get("date_extraction", []) + self._date_extraction_patterns = list(date_patterns_data) if isinstance(date_patterns_data, list) else [] + self._date_extraction_compiled = [re.compile(p) for p in self._date_extraction_patterns] + logger.info(f"Loaded {len(self._date_extraction_patterns)} date extraction patterns") + + # Load mappings (new format: list of {match, set} objects) + mappings_data = data.get("mappings", []) + self._mappings = [ + MappingRule.from_dict(m) for m in mappings_data + ] + logger.info(f"Loaded {len(self._mappings)} mapping rules: {self._mappings}") + + # Load templates + templates_data = data.get("templates", {}) + for name, config in templates_data.items(): + if isinstance(config, dict): + template = TemplateConfig.from_dict(name, config) + self._templates[name] = template + logger.info(f"Loaded template: {name}") + + def _load_legacy_format(self, data: dict[str, Any]) -> None: + """Load the legacy template-based configuration format.""" + logger.info("Loading legacy template-based configuration") + + if data and isinstance(data, dict): + for name, config in data.items(): + if isinstance(config, dict): + template = TemplateConfig.from_dict(name, config) + self._templates[name] = template + logger.info(f"Loaded legacy template: {name}") def load_from_directory(self, dirpath: str) -> None: - """Load templates from all YAML files in a directory. + """Load templates from all JSONC or YAML files in a directory. Args: - dirpath: Path to the directory containing YAML files. + dirpath: Path to the directory containing configuration files. """ path = Path(dirpath) if not path.exists(): - logger.warning(f"Template directory {dirpath} does not exist") + logger.warning(f"Configuration directory {dirpath} does not exist") return - for yaml_file in path.glob("*.yaml") + path.glob("*.yml"): - self.load_from_file(str(yaml_file)) + for config_file in path.glob("*.jsonc") + path.glob("*.json") + path.glob("*.yaml") + path.glob("*.yml"): + self.load_from_file(str(config_file)) - def get(self, name: str) -> TemplateConfig | None: + def is_new_format(self) -> bool: + """Check if the loaded configuration uses the new mapping-based format.""" + return self._is_new_format + + def get_companies(self) -> list[str]: + """Get the list of valid companies. + + Returns: + List of company names. + """ + return self._companies + + def get_document_types(self) -> list[str]: + """Get the list of valid document types. + + Returns: + List of document type names. + """ + return self._document_types + + def get_date_extraction_patterns(self) -> list[str]: + """Get the list of date extraction regex pattern strings. + + Returns: + List of regex pattern strings. + """ + return self._date_extraction_patterns + + def get_date_extraction_compiled(self) -> list[re.Pattern]: + """Get the list of compiled date extraction regex patterns. + + Returns: + List of compiled regex Pattern objects. + """ + return self._date_extraction_compiled + + def get_mapping_rules(self) -> list[MappingRule]: + """Get all mapping rules. + + Returns: + List of MappingRule objects. + """ + return self._mappings + + def resolve_mapping(self, company: str, document_type: str) -> dict[str, Any]: + """Resolve the mapping for a specific (company, document_type) tuple. + + Processes all rules in order. For each matching rule, applies the set fields. + Later matches override earlier ones. + + Args: + company: The company name. + document_type: The document type. + + Returns: + Dict with resolved fields: template, period_format, additional_fields. + If no mapping found, returns empty dict. + """ + result: dict[str, Any] = {} + + for rule in self._mappings: + if rule.matches(company, document_type): + # Apply set fields, overriding previous values + if rule.set.template is not None: + result["template"] = rule.set.template + if rule.set.period_format is not None: + result["period_format"] = rule.set.period_format + if rule.set.additional_fields: + result["additional_fields"] = rule.set.additional_fields.copy() + + return result + + def get_template(self, name: str) -> TemplateConfig | None: """Get a template by name. Args: @@ -221,7 +557,7 @@ class TemplateRegistry: """ return self._templates.get(name) - def list_all(self) -> list[str]: + def list_all_templates(self) -> list[str]: """List all template names. Returns: @@ -229,25 +565,71 @@ class TemplateRegistry: """ return list(self._templates.keys()) - def get_period_rule(self, template_name: str) -> "PeriodRule" | None: - """Get the period rule for a template. + def is_valid_company(self, company: str) -> bool: + """Check if a company is in the predefined list. + + Args: + company: The company name to check. + + Returns: + True if valid, False otherwise. + """ + return company in self._companies + + def is_valid_document_type(self, document_type: str) -> bool: + """Check if a document type is in the predefined list. + + Args: + document_type: The document type to check. + + Returns: + True if valid, False otherwise. + """ + return document_type in self._document_types + + def get_additional_fields(self) -> dict[str, list[str]]: + """Get the dictionary of additional field groups with their possible values. + + Returns: + Dict mapping field group names to list of possible values. + """ + return self._additional_fields + + def get_additional_field_values(self, group_name: str) -> list[str]: + """Get the list of possible values for a specific additional field group. + + Args: + group_name: The name of the additional field group. + + Returns: + List of possible values, or empty list if group not found. + """ + return self._additional_fields.get(group_name, []) + + def get_period_format_config(self, format_name: str) -> PeriodFormatConfig | None: + """Get the period format configuration by name. + + Args: + format_name: The name of the period format. + + Returns: + PeriodFormatConfig if found, None otherwise. + """ + return self._period_formats.get(format_name) + + def get_key_pattern(self, template_name: str) -> str | None: + """Get the key pattern for a template. Args: template_name: The template name. Returns: - PeriodRule, or None if not configured. + Key pattern string, or None if not found. """ - from .period import PeriodRule, PeriodGranularity, PeriodOffset - - template = self.get(template_name) - if not template or not template.period: - return None - - return PeriodRule( - granularity=PeriodGranularity(template.period.get("granularity", "month")), - offset=PeriodOffset(template.period.get("offset", "current")), - ) + template = self.get_template(template_name) + if template: + return template.key_pattern + return None def load_config() -> AppConfig: @@ -263,7 +645,8 @@ def load_templates(template_path: str | None = None) -> TemplateRegistry: """Load template configurations. Args: - template_path: Path to YAML file or directory. If None, uses TEMPLATE_PATH env var. + template_path: Path to JSONC file or directory. If None, uses TEMPLATE_PATH env var + or defaults to ./config.jsonc at the root. Returns: TemplateRegistry with loaded templates. @@ -271,7 +654,7 @@ def load_templates(template_path: str | None = None) -> TemplateRegistry: registry = TemplateRegistry() if template_path is None: - template_path = os.environ.get("TEMPLATE_PATH", "./templates.yaml") + template_path = os.environ.get("TEMPLATE_PATH", "./config.jsonc") if not template_path: logger.warning("No template path configured") diff --git a/automatic-pdf-renamer/app/period.py b/automatic-pdf-renamer/app/period.py index 0b386b1..1cd72dc 100644 --- a/automatic-pdf-renamer/app/period.py +++ b/automatic-pdf-renamer/app/period.py @@ -2,6 +2,12 @@ Provides period parsing from filenames and computation from emission dates per template rules (PRD section 4.1, FR4.2, FR4.3). + +Supports custom period format templates with placeholders: +- {year}, {year_next}, {year_previous} +- {month}, {month_name}, {month_abbr} +- {day} +- {quarter}, {semester}, {week} """ import re @@ -17,8 +23,10 @@ class PeriodGranularity(Enum): """Granularity of period computation (PRD section 4.1).""" DAY = "day" + WEEK = "week" MONTH = "month" QUARTER = "quarter" + SEMESTER = "semester" YEAR = "year" @@ -26,7 +34,8 @@ class PeriodOffset(Enum): """Offset for period computation (PRD section 4.1).""" CURRENT = "current" - NEXT = "next" + PREVIOUS = "previous" + FOLLOWING = "following" @dataclass @@ -47,43 +56,43 @@ class PeriodRule: # French month names mapping _FRENCH_MONTHS = { - "janvier": 1, - "février": 2, - "mars": 3, - "avril": 4, - "mai": 5, - "juin": 6, - "juillet": 7, - "août": 8, - "septembre": 9, - "octobre": 10, - "novembre": 11, - "décembre": 12, + "Janvier": 1, + "F\u00e9vrier": 2, + "Mars": 3, + "Avril": 4, + "Mai": 5, + "Juin": 6, + "Juillet": 7, + "Ao\u00fbt": 8, + "Septembre": 9, + "Octobre": 10, + "Novembre": 11, + "D\u00e9cembre": 12, # English fallbacks - "january": 1, - "february": 2, - "march": 3, - "april": 4, - "may": 5, - "june": 6, - "july": 7, - "august": 8, - "september": 9, - "october": 10, - "november": 11, - "december": 12, + "January": 1, + "February": 2, + "March": 3, + "April": 4, + "May": 5, + "June": 6, + "July": 7, + "August": 8, + "September": 9, + "October": 10, + "November": 11, + "December": 12, } # Month name to full French name for output _FRENCH_MONTH_NAMES = { - 1: "janvier", 2: "février", 3: "mars", 4: "avril", 5: "mai", 6: "juin", - 7: "juillet", 8: "août", 9: "septembre", 10: "octobre", 11: "novembre", 12: "décembre", + "01": "Janvier", "02": "F\u00e9vrier", "03": "Mars", "04": "Avril", "05": "Mai", "06": "Juin", + "07": "Juillet", "08": "Ao\u00fbt", "09": "Septembre", "10": "Octobre", "11": "Novembre", "12": "D\u00e9cembre", } # Month abbreviation mapping _MONTH_ABBR = { - 1: "Jan", 2: "Feb", 3: "Mar", 4: "Apr", 5: "May", 6: "Jun", - 7: "Jul", 8: "Aug", 9: "Sep", 10: "Oct", 11: "Nov", 12: "Dec", + "01": "Jan", "02": "Feb", "03": "Mar", "04": "Apr", "05": "May", "06": "Jun", + "07": "Jul", "08": "Aug", "09": "Sep", "10": "Oct", "11": "Nov", "12": "Dec", } # Quarter pattern - matches Q3 2025, T3 2025, 3 2025, Q3-2025, trimestre 3 2025, 3T2025, etc. @@ -97,7 +106,7 @@ _YEAR_PATTERN = re.compile(r"\b(\d{4})\b") # Month-year patterns (various formats) _MONTH_YEAR_PATTERNS = [ - # "août 2025", "August 2025" - French/English month name + year + # "ao\u00fbt 2025", "August 2025" - French/English month name + year re.compile( rf"\b([{''.join(_FRENCH_MONTHS.keys())}]+)\s+(\d{{4}})\b", re.IGNORECASE, @@ -138,7 +147,7 @@ def parse_period_from_filename(filename: str) -> str | None: Tries to extract period from the filename, looking at various positions. Supports formats: - - Month-year: "août 2025", "August 2025", "Aug 2025", "2025-08" + - Month-year: "ao\u00fbt 2025", "August 2025", "Aug 2025", "2025-08" - Quarter: "Q3 2025", "T3 2025", "3 2025", "Q3-2025" - Year: "2025" - Date: "2025-08-31", "31/08/2025" @@ -147,7 +156,7 @@ def parse_period_from_filename(filename: str) -> str | None: filename: The filename to parse (without path). Returns: - Parsed period string (e.g., "août 2025", "Q3 2025", "2025"), or None. + Parsed period string (e.g., "ao\u00fbt 2025", "Q3 2025", "2025"), or None. """ # Get the filename without path basename = filename.split("/")[-1].split("\\")[-1] @@ -227,6 +236,67 @@ def parse_period_from_filename(filename: str) -> str | None: return None +def _parse_date_string(date_str: str) -> date: + """Parse a date string into a date object. + + Supports various formats: YYYY-MM-DD, DD/MM/YYYY, MM/DD/YYYY, etc. + + Args: + date_str: The date string to parse. + + Returns: + date object. + + Raises: + ValueError: If the date string cannot be parsed. + """ + if isinstance(date_str, date): + return date_str + + # Try ISO format first + try: + return datetime.fromisoformat(date_str).date() + except ValueError: + pass + + # Try other common formats + formats = [ + "%Y-%m-%d", + "%d/%m/%Y", + "%m/%d/%Y", + "%d-%m-%Y", + "%Y/%m/%d", + "%Y-%m", + "%m-%Y", + "%Y", + ] + + for fmt in formats: + try: + return datetime.strptime(date_str, fmt).date() + except ValueError: + continue + + # Try to parse month name + year (e.g., "August 2025", "ao\u00fbt 2025") + month_year_match = re.match( + rf"^([{''.join(_FRENCH_MONTHS.keys())}]+)\s+(\d{{4}})$", + date_str.strip(), + re.IGNORECASE + ) + if month_year_match: + month_name = month_year_match.group(1) + year = int(month_year_match.group(2)) + month = _parse_french_month(month_name) + if month: + return date(year, month, 1) + + # Try year only + if re.match(r"^\d{4}$", date_str): + return date(int(date_str), 1, 1) + + raise ValueError(f"Cannot parse date string: {date_str}") + + def compute_period_from_date( emission_date: str | date, rule: PeriodRule, @@ -240,30 +310,18 @@ def compute_period_from_date( Returns: Human-readable period string. """ - if isinstance(emission_date, str): - # Parse ISO date - try: - dt = datetime.fromisoformat(emission_date).date() - except ValueError: - # Try other formats - for fmt in ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y"]: - try: - dt = datetime.strptime(emission_date, fmt).date() - break - except ValueError: - continue - else: - raise ValueError(f"Cannot parse emission_date: {emission_date}") - else: - dt = emission_date + dt = _parse_date_string(emission_date) if rule.granularity == PeriodGranularity.DAY: - return f"{dt.day} {_FRENCH_MONTH_NAMES[dt.month]} {dt.year}" + return f"{dt.day:02d} {_FRENCH_MONTH_NAMES[dt.month]} {dt.year:04d}" elif rule.granularity == PeriodGranularity.MONTH: if rule.offset == PeriodOffset.CURRENT: return format_month_year(dt.month, dt.year) - else: # NEXT + elif rule.offset == PeriodOffset.PREVIOUS: + prev_month = dt + relativedelta(months=-1) + return format_month_year(prev_month.month, prev_month.year) + else: # FOLLOWING next_month = dt + relativedelta(months=1) return format_month_year(next_month.month, next_month.year) @@ -271,7 +329,14 @@ def compute_period_from_date( quarter = (dt.month - 1) // 3 + 1 if rule.offset == PeriodOffset.CURRENT: return format_quarter(quarter, dt.year) - else: # NEXT + elif rule.offset == PeriodOffset.PREVIOUS: + prev_quarter = quarter - 1 + prev_year = dt.year + if prev_quarter < 1: + prev_quarter = 4 + prev_year -= 1 + return format_quarter(prev_quarter, prev_year) + else: # FOLLOWING next_quarter = quarter + 1 next_year = dt.year if next_quarter > 4: @@ -279,16 +344,223 @@ def compute_period_from_date( next_year += 1 return format_quarter(next_quarter, next_year) + elif rule.granularity == PeriodGranularity.SEMESTER: + semester = (dt.month - 1) // 6 + 1 + if rule.offset == PeriodOffset.CURRENT: + return f"S{semester} {dt.year:04d}" + elif rule.offset == PeriodOffset.PREVIOUS: + prev_semester = semester - 1 + prev_year = dt.year + if prev_semester < 1: + prev_semester = 2 + prev_year -= 1 + return f"S{prev_semester} {prev_year}" + else: # FOLLOWING + next_semester = semester + 1 + next_year = dt.year + if next_semester > 2: + next_semester = 1 + next_year += 1 + return f"S{next_semester} {next_year:04d}" + + elif rule.granularity == PeriodGranularity.WEEK: + # ISO week number + iso_year, iso_week, _ = dt.isocalendar() + if rule.offset == PeriodOffset.CURRENT: + return f"Week {iso_week:02d} {iso_year:04d}" + elif rule.offset == PeriodOffset.PREVIOUS: + prev_date = dt - relativedelta(weeks=1) + prev_year, prev_week, _ = prev_date.isocalendar() + return f"Week {prev_week:02d} {prev_year:04d}" + else: # FOLLOWING + next_date = dt + relativedelta(weeks=1) + next_year, next_week, _ = next_date.isocalendar() + return f"Week {next_week:02d} {next_year:04d}" + elif rule.granularity == PeriodGranularity.YEAR: if rule.offset == PeriodOffset.CURRENT: return str(dt.year) - else: # NEXT + elif rule.offset == PeriodOffset.PREVIOUS: + return str(dt.year - 1) + else: # FOLLOWING return str(dt.year + 1) else: raise ValueError(f"Unknown granularity: {rule.granularity}") +def format_period_from_emission_date( + emission_date: str | date, + period_format: dict[str, Any], +) -> str: + """Format period from emission date using custom period format configuration. + + The period_format dict contains: + - granularity: day, week, month, quarter, semester, year + - format: Custom template string with placeholders + - offset: current, previous, following + + Supported placeholders in format template: + - {year}: Current year + - {year_next}: Next year + - {year_previous}: Previous year + - {month}: Month number (1-12) + - {month_name}: Full month name (French) + - {month_abbr}: 3-letter month abbreviation + - {day}: Day of month (1-31) + - {quarter}: Quarter number (1-4) + - {semester}: Semester number (1-2) + - {week}: ISO week number (1-53) + + Args: + emission_date: The document emission date (string or date object). + period_format: Dict with granularity, format, and offset keys. + + Returns: + Formatted period string with placeholders replaced. + + Raises: + ValueError: If granularity is unknown or format template is invalid. + """ + dt = _parse_date_string(emission_date) + + granularity = period_format.get("granularity", "month") + format_template = period_format.get("format", "") + offset = period_format.get("offset", "current") + + # Determine the base date based on offset + if offset == "previous": + # Date is at the end of the period, so we need the previous period + if granularity == "day": + base_date = dt - relativedelta(days=1) + elif granularity == "week": + base_date = dt - relativedelta(weeks=1) + elif granularity == "month": + base_date = dt - relativedelta(months=1) + elif granularity == "quarter": + quarter = (dt.month - 1) // 3 + 1 + if quarter == 1: + base_date = date(dt.year - 1, 12, 1) + else: + base_date = date(dt.year, (quarter - 1) * 3, 1) + elif granularity == "semester": + semester = (dt.month - 1) // 6 + 1 + if semester == 1: + base_date = date(dt.year - 1, 12, 1) + else: + base_date = date(dt.year, 6, 1) + elif granularity == "year": + base_date = date(dt.year - 1, 1, 1) + else: + base_date = dt + elif offset == "following": + # Date is at the beginning of the period, so we need the following period + if granularity == "day": + base_date = dt + relativedelta(days=1) + elif granularity == "week": + base_date = dt + relativedelta(weeks=1) + elif granularity == "month": + base_date = dt + relativedelta(months=1) + elif granularity == "quarter": + quarter = (dt.month - 1) // 3 + 1 + if quarter == 4: + base_date = date(dt.year + 1, 1, 1) + else: + base_date = date(dt.year, (quarter + 1) * 3, 1) + elif granularity == "semester": + semester = (dt.month - 1) // 6 + 1 + if semester == 2: + base_date = date(dt.year + 1, 1, 1) + else: + base_date = date(dt.year, 6, 1) + elif granularity == "year": + base_date = date(dt.year + 1, 1, 1) + else: + base_date = dt + else: # current + # Date is in the middle of the period, use the date as-is + base_date = dt + + # Calculate period values based on granularity + period_values: dict[str, Any] = {} + + if granularity == "day": + period_values["year"] = f"{base_date.year:04d}" + period_values["month"] = f"{base_date.month:02d}" + period_values["month_name"] = _FRENCH_MONTH_NAMES[f"{base_date.month:02d}"] + period_values["month_abbr"] = _MONTH_ABBR[f"{base_date.month:02d}"] + period_values["day"] = f"{base_date.day:02d}" + period_values["quarter"] = (base_date.month - 1) // 3 + 1 + period_values["semester"] = (base_date.month - 1) // 6 + 1 + _, period_values["week"], _ = base_date.isocalendar() + + elif granularity == "week": + _, period_values["week"], _ = base_date.isocalendar() + period_values["year"] = f"{base_date.year:04d}" + period_values["month"] = f"{base_date.month:02d}" + period_values["month_name"] = _FRENCH_MONTH_NAMES[f"{base_date.month:02d}"] + period_values["month_abbr"] = _MONTH_ABBR[f"{base_date.month:02d}"] + period_values["day"] = f"{base_date.day:02d}" + period_values["quarter"] = (base_date.month - 1) // 3 + 1 + period_values["semester"] = (base_date.month - 1) // 6 + 1 + + elif granularity == "month": + period_values["year"] = f"{base_date.year:04d}" + period_values["month"] = f"{base_date.month:02d}" + period_values["month_name"] = _FRENCH_MONTH_NAMES[f"{base_date.month:02d}"] + period_values["month_abbr"] = _MONTH_ABBR[f"{base_date.month:02d}"] + period_values["day"] = f"{1:02d}" # Default to first day of month + period_values["quarter"] = (base_date.month - 1) // 3 + 1 + period_values["semester"] = (base_date.month - 1) // 6 + 1 + _, period_values["week"], _ = base_date.isocalendar() + + elif granularity == "quarter": + quarter = (base_date.month - 1) // 3 + 1 + period_values["quarter"] = quarter + period_values["year"] = f"{base_date.year:04d}" + period_values["month"] = f"{(quarter - 1) * 3 + 1:02d}" # First month of quarter + period_values["month_name"] = _FRENCH_MONTH_NAMES[f"{period_values['month']}"] + period_values["month_abbr"] = _MONTH_ABBR[f"{period_values['month']}"] + period_values["day"] = f"{1:02d}" + period_values["semester"] = (quarter - 1) // 2 + 1 + _, period_values["week"], _ = base_date.isocalendar() + + elif granularity == "semester": + semester = (base_date.month - 1) // 6 + 1 + period_values["semester"] = semester + period_values["year"] = f"{base_date.year:04d}" + period_values["month"] = f"{(semester - 1) * 6 + 1:02d}" # First month of semester + period_values["month_name"] = _FRENCH_MONTH_NAMES[f"{period_values['month']}"] + period_values["month_abbr"] = _MONTH_ABBR[f"{period_values['month']}"] + period_values["day"] = f"{1:02d}" + period_values["quarter"] = (period_values["month"] - 1) // 3 + 1 + _, period_values["week"], _ = base_date.isocalendar() + + elif granularity == "year": + period_values["year"] = f"{base_date.year:04d}" + period_values["month"] = f"{1:02d}" + period_values["month_name"] = "janvier" + period_values["month_abbr"] = "Jan" + period_values["day"] = f"{1:02d}" + period_values["quarter"] = 1 + period_values["semester"] = 1 + _, period_values["week"], _ = base_date.isocalendar() + + else: + raise ValueError(f"Unknown granularity: {granularity}") + + # Add year_next and year_previous + period_values["year_next"] = f"{int(period_values['year']) + 1:04d}" + period_values["year_previous"] = f"{int(period_values['year']) - 1:04d}" + + # Format the template + try: + return format_template.format(**period_values) + except KeyError as e: + missing = str(e).strip("'") + raise ValueError(f"Unknown placeholder '{missing}' in format template: {format_template}") + + def format_month_year(month: int, year: int, use_french: bool = True) -> str: """Format month and year as a human-readable string. @@ -300,7 +572,7 @@ def format_month_year(month: int, year: int, use_french: bool = True) -> str: use_french: If True, use French month names; otherwise use English. Returns: - Formatted string (e.g., "août 2025" or "August 2025"). + Formatted string (e.g., "ao\u00fbt 2025" or "August 2025"). """ if month < 1 or month > 12: raise ValueError(f"Invalid month: {month}") @@ -308,7 +580,7 @@ def format_month_year(month: int, year: int, use_french: bool = True) -> str: month_name = _FRENCH_MONTH_NAMES[month] else: month_name = _MONTH_ABBR[month] - return f"{month_name} {year}" + return f"{month_name} {year:04d}" def format_quarter(quarter: int, year: int) -> str: @@ -323,7 +595,7 @@ def format_quarter(quarter: int, year: int) -> str: """ if quarter < 1 or quarter > 4: raise ValueError(f"Invalid quarter: {quarter}") - return f"Q{quarter} {year}" + return f"Q{quarter} {year:04d}" def get_period( @@ -361,20 +633,7 @@ def get_period( # If we have emission_date but no rule, use month-year as fallback if emission_date: - if isinstance(emission_date, str): - try: - dt = datetime.fromisoformat(emission_date).date() - except ValueError: - for fmt in ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y"]: - try: - dt = datetime.strptime(emission_date, fmt).date() - break - except ValueError: - continue - else: - raise ValueError(f"Cannot parse emission_date: {emission_date}") - else: - dt = emission_date + dt = _parse_date_string(emission_date) return (format_month_year(dt.month, dt.year), False) raise ValueError("Cannot determine period: no filename, emission_date, or rule provided") @@ -396,20 +655,7 @@ def validate_period_consistency( Tuple of (is_consistent, error_message). If consistent, error_message is None. """ - if isinstance(emission_date, str): - try: - dt = datetime.fromisoformat(emission_date).date() - except ValueError: - for fmt in ["%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y"]: - try: - dt = datetime.strptime(emission_date, fmt).date() - break - except ValueError: - continue - else: - return (False, f"Cannot parse emission_date: {emission_date}") - else: - dt = emission_date + dt = _parse_date_string(emission_date) # Compute expected period from rule expected_period = compute_period_from_date(dt, rule) diff --git a/automatic-pdf-renamer/app/pipeline.py b/automatic-pdf-renamer/app/pipeline.py new file mode 100644 index 0000000..e132e1a --- /dev/null +++ b/automatic-pdf-renamer/app/pipeline.py @@ -0,0 +1,558 @@ +"""Processing pipeline module for Automatic PDF Renamer. + +Implements the document processing workflow: +1. Load the first page and extract text using PyMuPDF +2. Find the emission date using regex patterns from config (ordered list, first match wins) +3. Find company and document_type using ML, validated against predefined lists in config + - Unknown values must route to review +4. Look up the (company, document_type) tuple in mappings to get template, period_format, and additional_fields + - Mappings use match/set structure: match criteria (company, document_type, both, or neither) + - Rules are processed in order, later matches override earlier ones + - Empty match {} matches all documents (default rule) +5. Apply period_format (with granularity, format template, and offset) to calculate the human-readable period string +6. Apply template key_pattern to generate the final filename +""" + +import logging +import re +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any + +from .config import ( + TemplateRegistry, + MappingRule, + PeriodFormatConfig, +) +from .extractor import extract_first_page_text, is_text_empty, TextExtractionError +from .period import ( + format_period_from_emission_date, +) +from .sidecar import Sidecar, SidecarStatus, build_sidecar + +logger = logging.getLogger(__name__) + + +@dataclass +class ExtractedDate: + """Represents an extracted date from text.""" + + date_str: str + datetime_obj: datetime | None + pattern_index: int = -1 + + +@dataclass +class ProcessingResult: + """Result of processing a document.""" + + success: bool + sidecar: Sidecar | None = None + errors: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + emission_date: str = "" + company: str = "" + document_type: str = "" + template_name: str = "" + period: str = "" + status: str = SidecarStatus.NOT_PROCESSED.value + + +class DocumentProcessor: + """Processes PDF documents through the pipeline.""" + + def __init__( + self, + registry: TemplateRegistry, + min_text_length: int = 10, + ): + """Initialize the document processor. + + Args: + registry: TemplateRegistry with configuration. + min_text_length: Minimum text length for valid extraction. + """ + self.registry = registry + self.min_text_length = min_text_length + + def process( + self, + pdf_bytes: bytes, + original_key: str, + current_key: str, + sha256: str, + extracted_text: str | None = None, + ) -> ProcessingResult: + """Process a PDF document through the full pipeline. + + Args: + pdf_bytes: The PDF content as bytes. + original_key: The original S3 key of the file. + current_key: The current S3 key of the file. + sha256: The SHA256 hash of the PDF. + extracted_text: Optional pre-extracted text (for caching). + + Returns: + ProcessingResult with the processing outcome. + """ + result = ProcessingResult(success=False) + + try: + # Step 1: Extract first page text + if extracted_text is None: + extracted_text = self._extract_text(pdf_bytes) + + if extracted_text is None: + result.errors.append("Failed to extract text from PDF") + result.status = SidecarStatus.NOT_PROCESSED.value + return result + + # Step 2: Find emission date using regex patterns from config + emission_date, date_error = self._extract_emission_date(extracted_text) + if date_error: + result.errors.append(date_error) + result.status = SidecarStatus.NOT_PROCESSED.value + return result + + result.emission_date = emission_date + + # Step 3: Find company and document_type using ML + company, doc_type, ml_errors = self._extract_company_and_type(extracted_text) + + if ml_errors: + result.errors.extend(ml_errors) + result.status = SidecarStatus.NOT_PROCESSED.value + return result + + result.company = company + result.document_type = doc_type + + # Validate company and document_type against config lists + # Strict validation: unknown values route to review + if not self.registry.is_valid_company(company): + result.errors.append( + f"Unknown company '{company}'. " + f"Valid companies: {', '.join(self.registry.get_companies()[:10])}..." + ) + result.status = SidecarStatus.NOT_PROCESSED.value + return result + + if not self.registry.is_valid_document_type(doc_type): + result.errors.append( + f"Unknown document type '{doc_type}'. " + f"Valid types: {', '.join(self.registry.get_document_types()[:10])}..." + ) + result.status = SidecarStatus.NOT_PROCESSED.value + return result + + # Step 4: Resolve mapping using match/set rules + # Rules are processed in order, later matches override earlier ones + resolved = self.registry.resolve_mapping(company, doc_type) + + if not resolved: + result.errors.append( + f"No mapping found for company='{company}', " + f"document_type='{doc_type}'. Routing to review." + ) + result.status = SidecarStatus.NOT_PROCESSED.value + return result + + # Extract resolved fields + template_name = resolved.get("template") + period_format_name = resolved.get("period_format") + additional_field_groups = resolved.get("additional_fields", []) + + if not template_name: + result.errors.append( + f"No template resolved for company='{company}', " + f"document_type='{doc_type}'. Routing to review." + ) + result.status = SidecarStatus.NOT_PROCESSED.value + return result + + result.template_name = template_name + + # Step 4b: Extract additional fields from additional_fields lists + optional_fields = {} + for group_name in additional_field_groups: + field_value, field_error = self._extract_field_value( + extracted_text, group_name + ) + if field_error: + result.errors.append(field_error) + result.status = SidecarStatus.NOT_PROCESSED.value + return result + if field_value: + optional_fields[group_name] = field_value + + # Step 5: Calculate period using period_format configuration + if not period_format_name: + result.errors.append( + f"No period_format resolved for company='{company}', " + f"document_type='{doc_type}'. Routing to review." + ) + result.status = SidecarStatus.NOT_PROCESSED.value + return result + + period_format_config = self.registry.get_period_format_config(period_format_name) + if period_format_config is None: + result.errors.append( + f"Period format '{period_format_name}' not found in configuration" + ) + result.status = SidecarStatus.NOT_PROCESSED.value + return result + + try: + period = format_period_from_emission_date( + emission_date, + period_format_config.to_dict() + ) + result.period = period + except Exception as e: + result.errors.append(f"Failed to compute period: {e}") + result.status = SidecarStatus.NOT_PROCESSED.value + return result + + # Step 6: Generate the final key using template's key_pattern + key_pattern = self.registry.get_key_pattern(template_name) + if key_pattern: + try: + final_key = self._format_key_pattern( + key_pattern, + company=company, + document_type=doc_type, + emission_date=emission_date, + period=period, + template_name=template_name, + emitting_company=company, + original_filename=original_key.split("/")[-1], + **optional_fields, + ) + except Exception as e: + result.errors.append(f"Failed to format key pattern: {e}") + result.status = SidecarStatus.NOT_PROCESSED.value + return result + else: + # Fallback: use a simple pattern + final_key = f"{template_name}/{period}/{company}/{doc_type}/{emission_date}_{original_key.split('/')[-1]}" + + # Step 7: Build the sidecar + confidence = 1.0 # Will be set by ML classifier + sidecar = build_sidecar( + sha256=sha256, + original_key=original_key, + current_key=final_key, + status=SidecarStatus.PROCESSED, + template_name=template_name, + emission_date=emission_date, + period=period, + emitting_company=company, + document_type=doc_type, + confidence=confidence, + optional_fields=optional_fields, + extracted_text=extracted_text, + ) + + # Add processing event + sidecar.add_event( + "classified", + { + "template": template_name, + "company": company, + "document_type": doc_type, + "emission_date": emission_date, + "period": period, + "confidence": confidence, + "optional_fields": optional_fields, + }, + ) + + result.success = True + result.sidecar = sidecar + result.status = SidecarStatus.PROCESSED.value + + except Exception as e: + result.errors.append(f"Unexpected error during processing: {e}") + result.status = SidecarStatus.NOT_PROCESSED.value + logger.exception(f"Error processing document {original_key}") + + return result + + def _extract_text(self, pdf_bytes: bytes) -> str | None: + """Extract first page text from PDF. + + Args: + pdf_bytes: The PDF content as bytes. + + Returns: + Extracted text, or None if extraction fails. + """ + try: + text = extract_first_page_text(pdf_bytes) + if is_text_empty(text, self.min_text_length): + return None + return text + except TextExtractionError as e: + logger.error(f"Text extraction failed: {e}") + return None + + def _extract_emission_date(self, text: str) -> tuple[str, str | None]: + """Extract emission date from text using regex patterns from config. + + Uses ordered list of regex patterns. Tries them in order and stops + at first match. + + Args: + text: The extracted text from the PDF. + + Returns: + Tuple of (date_string, error_message). + If successful, error_message is None. + """ + patterns = self.registry.get_date_extraction_compiled() + + if not patterns: + # Fallback: try to find any date-like pattern + return self._extract_emission_date_fallback(text) + + # Try each pattern in order, first match wins + for idx, pattern in enumerate(patterns): + matches = pattern.finditer(text) + for match in matches: + # Get the first date-like match + groups = match.groups() + date_str = self._reconstruct_date(groups) + if date_str: + # Validate it looks like a date + if self._is_valid_date_string(date_str): + return (date_str, None) + + # If no patterns matched, try fallback + return self._extract_emission_date_fallback(text) + + def _reconstruct_date(self, groups: tuple[str, ...]) -> str | None: + """Reconstruct a date string from regex match groups. + + Args: + groups: The regex match groups. + + Returns: + Reconstructed date string, or None if cannot reconstruct. + """ + # Filter out None and empty strings + non_empty = [g for g in groups if g] + if non_empty: + return " ".join(non_empty) if len(non_empty) > 1 else non_empty[0] + return None + + def _is_valid_date_string(self, date_str: str) -> bool: + """Check if a string looks like a valid date. + + Args: + date_str: The string to check. + + Returns: + True if it looks like a valid date. + """ + # Simple heuristic: contains digits and possibly separators + if not date_str: + return False + + # Check for common date patterns + date_patterns = [ + r"\d{4}-\d{2}-\d{2}", # YYYY-MM-DD + r"\d{2}/\d{2}/\d{4}", # DD/MM/YYYY + r"\d{2}-\d{2}-\d{4}", # DD-MM-YYYY + r"\d{4}", # YYYY + r"\d{4}/\d{2}", # YYYY/MM + r"\d{2}\s+[a-zA-Z]+\s+\d{4}", # DD Month YYYY + ] + + for p in date_patterns: + if re.match(p, date_str.strip()): + return True + + return False + + def _extract_emission_date_fallback(self, text: str) -> tuple[str, str | None]: + """Fallback method for extracting emission date. + + Args: + text: The extracted text. + + Returns: + Tuple of (date_string, error_message). + """ + # Try common date patterns + fallback_patterns = [ + r"(\d{4}-\d{2}-\d{2})", # YYYY-MM-DD + r"(\d{2}/\d{2}/\d{4})", # DD/MM/YYYY + r"(\d{2}-\d{2}-\d{4})", # DD-MM-YYYY + r"(\d{4})", # YYYY + ] + + for pattern in fallback_patterns: + matches = re.finditer(pattern, text) + for match in matches: + date_str = match.group(1) + if self._is_valid_date_string(date_str): + return (date_str, None) + + return ("", "Could not find emission date in text") + + def _extract_company_and_type(self, text: str) -> tuple[str, str, list[str]]: + """Extract company and document type from text using ML. + + This is a placeholder for the actual ML implementation. + For now, it tries to match company and document type names from config. + + Args: + text: The extracted text. + + Returns: + Tuple of (company, document_type, errors). + """ + errors = [] + + # Try to find company names from the config + companies = self.registry.get_companies() + for company in companies: + if company.lower() in text.lower(): + # Try to find document type + doc_types = self.registry.get_document_types() + for doc_type in doc_types: + if doc_type.lower() in text.lower(): + return (company, doc_type, errors) + + # If no document type found, use first one as default + if doc_types: + return (company, doc_types[0], [f"Could not determine document type, using default: {doc_types[0]}"]) + + # If no company found, return unknown + return ("", "", ["Could not determine company and document type"]) + + def _extract_field_value(self, text: str, group_name: str) -> tuple[str, str | None]: + """Extract a field value from text using ML from predefined list. + + For each additional field group in the mapping, the value should be found + using ML from the predefined list of possible values for that group. + + Strict validation: if value not found in predefined list, route to review. + + Args: + text: The extracted text. + group_name: The name of the additional field group. + + Returns: + Tuple of (field_value, error_message). + If successful, error_message is None. + If no value can be determined, error_message describes the issue. + """ + # Get the predefined list of values for this additional field group + possible_values = self.registry.get_additional_field_values(group_name) + + if not possible_values: + return ("", f"No predefined values for additional field group '{group_name}'") + + # Try to find a value from the predefined list in the text + # This is a placeholder for ML-based extraction + # For now, we'll do simple string matching + text_lower = text.lower() + for value in possible_values: + if value.lower() in text_lower: + return (value, None) + + # Strict validation: if no value found in predefined list, route to review + return ("", f"Could not determine value for additional field '{group_name}' from predefined list") + + def _format_key_pattern( + self, + pattern: str, + **kwargs: Any, + ) -> str: + """Format a key pattern with the provided values. + + Args: + pattern: The key pattern with placeholders. + **kwargs: Values to substitute into the pattern. + + Returns: + Formatted key string. + """ + try: + return pattern.format(**kwargs) + except KeyError as e: + # If a placeholder is missing, use a default value + missing = str(e).strip("'") + kwargs[missing] = "unknown" + return pattern.format(**kwargs) + + +class DateExtractor: + """Specialized class for extracting dates from text using config patterns.""" + + def __init__(self, registry: TemplateRegistry): + """Initialize the date extractor. + + Args: + registry: TemplateRegistry with date extraction patterns. + """ + self.registry = registry + self._patterns = registry.get_date_extraction_compiled() + + def extract_first_date(self, text: str) -> str | None: + """Extract the first date found in text using configured patterns. + + Args: + text: The text to search. + + Returns: + The first date string found, or None. + """ + if not self._patterns: + return None + + for pattern in self._patterns: + match = pattern.search(text) + if match: + groups = match.groups() + date_str = " ".join([g for g in groups if g]) + if date_str.strip(): + return date_str.strip() + + return None + + def extract_all_dates(self, text: str) -> list[str]: + """Extract all dates found in text using configured patterns. + + Args: + text: The text to search. + + Returns: + List of date strings found. + """ + dates = [] + if not self._patterns: + return dates + + for pattern in self._patterns: + matches = pattern.finditer(text) + for match in matches: + groups = match.groups() + date_str = " ".join([g for g in groups if g]) + if date_str.strip() and date_str.strip() not in dates: + dates.append(date_str.strip()) + + return dates + + +def create_processor(registry: TemplateRegistry) -> DocumentProcessor: + """Factory function to create a DocumentProcessor. + + Args: + registry: TemplateRegistry with configuration. + + Returns: + DocumentProcessor instance. + """ + return DocumentProcessor(registry) diff --git a/automatic-pdf-renamer/arch.md b/automatic-pdf-renamer/arch.md index 7c07d18..54d2d43 100644 --- a/automatic-pdf-renamer/arch.md +++ b/automatic-pdf-renamer/arch.md @@ -41,7 +41,7 @@ flowchart TD - Offline-first: no runtime downloads, no external services (NFR1, AC10). - Zero silent wrong renames: every ambiguous path routes to not processed (NFR3). - Crash safety: copy-then-delete renames, atomic sidecar writes, SQLite repairable from sidecars alone (NFR4, AC8, AC9). -- Minimal operational surface: one container, one YAML template config, env-var credentials (NFR6). +- Minimal operational surface: one container, one JSONC configuration file, env-var credentials (NFR6). - Single user, local network: performance targets are modest (NFR2) but correctness targets are strict. ## 3. Tech Stack and Key Decisions @@ -80,14 +80,13 @@ On startup, the process runs the recovery scan (FR1.3, FR10.3): objects in `inbo ``` app/ main.py # FastAPI app, lifespan: recovery scan + poller start - config.py # env vars + YAML template loading + config.py # env vars + JSONC configuration loading storage/ # S3 interface (boto3), key helpers, collision suffix sidecar.py # atomic sidecar write (temp key + copy), schema replica/ # SQLAlchemy models, sync, rebuild pipeline/ # extract (PyMuPDF), classify, fill fields, validate - period.py # period computation from emission date + template rules - gazetteer.py # counterparty matching (rapidfuzz) - templates.py # YAML template registry (slots, period rules, key patterns) + period.py # period computation from emission date + period format config + templates.py # JSONC template registry (slots, period rules, key patterns) rename.py # copy-then-delete move + event logging jobs/ # poller, consistency scan, retrain subprocess, rebuild api/ # routes: files, templates, retrain, consistency, auth @@ -108,7 +107,7 @@ flowchart TD empty -- no --> cls["Classify (TF-IDF + LogReg)"] cls --> conf{"Confidence >= threshold?"} conf -- no --> pending - conf -- yes --> fill["Fill fields: regexes, gazetteer, period from emission date"] + conf -- yes --> fill["Fill fields: regexes, companies list, additional fields from groups, period from emission date"] fill --> valid{"Fields valid?"} valid -- no --> pending valid -- yes --> rename["Compute key, copy-then-delete, write sidecar + SQLite"] @@ -172,7 +171,7 @@ Subprocesses report progress and metrics by writing a status JSON consumed by th ## 7. Configuration and Secrets -- Templates: single YAML file (or a directory of YAML files) defining, per template: document family, document types, period rules, optional field schema, extraction regexes, validation rules, key pattern, target prefix. +- Templates: single JSONC file defining: document family, document types, period rules, optional field schema, extraction regexes, validation rules, key pattern, target prefix. - Environment: S3 endpoint, access key, secret key, bucket name, poll interval, confidence threshold, dashboard credentials, paths (SQLite file, model registry, template YAML). - No secrets in the YAML; no secrets in sidecars. diff --git a/automatic-pdf-renamer/config.jsonc b/automatic-pdf-renamer/config.jsonc new file mode 100644 index 0000000..0e9e15e --- /dev/null +++ b/automatic-pdf-renamer/config.jsonc @@ -0,0 +1,178 @@ +{ + "version": "1.0", + "companies": [ + "EDF", + "Engie", + "BNP Paribas", + "Societe Generale", + "Credit Agricole", + "La Banque Postale", + "Orange", + "SFR", + "Free", + "Bouygues Telecom", + "TotalEnergies", + "ENEDIS", + "GRDF", + "Veolia", + "Suez" + ], + "document_types": [ + "facture", + "releve", + "contrat", + "avis" + ], + "additional_fields": { + "bank_accounts": [ + "FR76 1234 5678 9012 3456 7890 123", + "FR76 2345 6789 0123 4567 8901 234", + "AMBIGUOUS_VALUE" + ], + "properties": [ + "123 Rue de Paris, 75001 Paris", + "456 Avenue des Champs, 75002 Paris" + ], + "contract_references": [ + "CONT-2024-001", + "CONT-2024-002", + "AMBIGUOUS_VALUE" + ], + "customer_numbers": [ + "CUST-12345678", + "CUST-87654321" + ] + }, + "date_extraction": [ + "([0-9]{1,2})[/-]([0-9]{1,2})[/-]([0-9]{4})", + "([0-9]{4})[-/]([0-9]{1,2})[-/]([0-9]{1,2})", + "([0-9]{1,2}) (janvier|fevrier|mars|avril|mai|juin|juillet|aout|septembre|octobre|novembre|decembre) ([0-9]{4})", + "(janvier|fevrier|mars|avril|mai|juin|juillet|aout|septembre|octobre|novembre|decembre) ([0-9]{4})", + "([0-9]{4})" + ], + "period_formats": { + "monthly": { + "granularity": "month", + "format": "{month_name} {year}", + "offset": "current" + }, + "monthly_numeric": { + "granularity": "month", + "format": "{year}-{month}", + "offset": "current" + }, + "quarterly": { + "granularity": "quarter", + "format": "T{quarter} {year}", + "offset": "current" + }, + "quarterly_q": { + "granularity": "quarter", + "format": "Q{quarter} {year}", + "offset": "current" + }, + "semester": { + "granularity": "semester", + "format": "S{semester} {year}", + "offset": "current" + }, + "yearly": { + "granularity": "year", + "format": "{year}", + "offset": "current" + } + }, + "mappings": [ + { + "match": {}, + "set": { + "template": "default", + "period_format": "monthly" + } + }, + { + "match": { + "document_type": "releve" + }, + "set": { + "template": "bank_statement", + "additional_fields": [ + "bank_accounts" + ] + } + }, + { + "match": { + "company": "BNP Paribas" + }, + "set": { + "period_format": "monthly" + } + }, + { + "match": { + "document_type": "contrat" + }, + "set": { + "template": "contract", + "period_format": "yearly" + } + }, + { + "match": { + "company": "EDF", + "document_type": "facture" + }, + "set": { + "template": "edf_invoice", + "period_format": "monthly", + "additional_fields": [ + "contract_references", + "customer_numbers" + ] + } + }, + { + "match": { + "company": "EDF", + "document_type": "avis" + }, + "set": { + "template": "edf_notice", + "period_format": "quarterly_q" + } + }, + { + "match": { + "company": "Veolia", + "document_type": "facture" + }, + "set": { + "template": "water_invoice", + "additional_fields": [ + "properties" + ] + } + } + ], + "templates": { + "default": { + "key_pattern": "{emitting_company}/{period}/{document_type}/{emission_date}_{original_filename}" + }, + "edf_invoice": { + "key_pattern": "{emitting_company}/{period}/{document_type}/{emission_date}_{original_filename}" + }, + "edf_notice": { + "key_pattern": "{emitting_company}/{period}/{document_type}/{emission_date}_{original_filename}" + }, + "bank_statement": { + "key_pattern": "Banques/{emitting_company}/{period}/{document_type}/{emission_date}_{original_filename}" + }, + "contract": { + "key_pattern": "Contrats/{emitting_company}/{period}/{document_type}/{emission_date}_{original_filename}" + }, + "water_invoice": { + "key_pattern": "Eau/{emitting_company}/{period}/{document_type}/{emission_date}_{original_filename}" + } + } +} diff --git a/automatic-pdf-renamer/prd.md b/automatic-pdf-renamer/prd.md index 1e06f07..a19d16c 100644 --- a/automatic-pdf-renamer/prd.md +++ b/automatic-pdf-renamer/prd.md @@ -82,23 +82,19 @@ Every document, regardless of template, carries at least these five common field - Common field 1 — Template: the selected template (i.e. the document family: invoice, purchase order, contract, bank statement, rent receipt, etc.). - Common field 2 — Emission date: the date the document was emitted, extracted from its content (not the object modification date). -- Common field 3 — Emitting company: the company or organization that emitted the document, matched against the counterparty gazetteer. +- Common field 3 — Emitting company: the company or organization that emitted the document, matched against the companies list in configuration. - Common field 4 — Document type: the fine-grained type of document within the family (e.g. debit advice vs monthly statement for bank statements; quotes vs order confirmations for purchase documents). -- Common field 5 — Period: the period the document refers to, in human-readable form, derived from the emission date according to the template's period rules. The granularity and the offset relative to the emission date are declared per template: - - Granularity `day`: the emission date itself, e.g. "20 September 2026" (a simple invoice). - - Granularity `month`: e.g. "September 2026" (a monthly rent receipt). - - Granularity `quarter`: e.g. "Q3 2026" (quarterly building-charge invoice). - - Granularity `year`: e.g. "2026" (yearly tax document). - - Offset `current`: the period containing the emission date (document sent after the start of the period; e.g. "Q3 2026" for 2026-09-20). - - Offset `next`: the period following the emission date's period (document sent in advance; e.g. "Q4 2026" for 2026-09-20). - - Offset is only meaningful for granularities coarser than a day; for `day` the period always equals the emission date. +- Common field 5 u2014 Period: the period the document refers to, in human-readable form, derived from the emission date according to the period format configuration. The period format is selected per (company, document_type) mapping and includes: + - Granularity: `day`, `week`, `month`, `quarter`, `semester`, or `year` + - Format: custom template string with placeholders (e.g., `{month_name} {year}`, `Q{quarter} {year}`) + - Offset: `current` (date in middle of period), `previous` (arrives after period), `following` (arrives before period) The object key pattern of each template is composed from these common fields, plus the template's optional fields. #### 4.2 Optional fields (template-specific) -- FR4.1 Each template declares in YAML its optional fields beyond the common five: name, type, format, extraction rules, validation rules, and whether the field is rendered in the object key, the target prefix, or stored only. Examples: bank statements carry account owner and statement type; rent receipts carry the concerned premises; contracts carry the counterparty signatory. -- FR4.2 Fill common and optional fields using per-template regexes, dateutil for dates, and a counterparty gazetteer with rapidfuzz matching for the emitting company (similarity threshold configurable; below threshold the company is "unrecognized"). The period is computed from the emission date using the template's declared granularity and offset (current vs next period); it is recomputed whenever the emission date or template changes. +- FR4.1 Each template declares in JSONC its optional fields beyond the common five. Additional fields are organized in predefined field groups (e.g., `bank_accounts`, `properties`, `contract_references`, `customer_numbers`) with their possible values. The (company, document_type) mapping specifies which field groups to extract for each document. Examples: bank statements extract from `bank_accounts` group; rent receipts extract from `properties` group. +- FR4.2 Fill common and optional fields using per-template regexes for dates and the companies list for company matching. Additional fields are extracted from their predefined field groups using ML from the possible values. The period is computed from the emission date using the period format's granularity, format template, and offset (current/previous/following); it is recomputed whenever the emission date or period format changes. - FR4.3 Validate fields before renaming: emission date parses and is within a plausible range, period is non-empty and consistent with the emission date and the template's granularity/offset, emitting company recognized, document type among the template's allowed values, each optional field matches its declared format (empty optional fields are allowed). Any failure routes to not processed with the failing fields identified. - FR4.4 Collisions in the target prefix are resolved with an incremental suffix; the final key is always the one logged. @@ -156,11 +152,11 @@ The object key pattern of each template is composed from these common fields, pl Sidecar JSON (source of truth), one per PDF at `{key}.meta.json`: -- schema_version, sha256, original_key, current_key, status, template_name, confidence, emission_date, period, emitting_company, document_type, optional_field1, optional_field2, extracted_text, created_at, updated_at, events[] (type: classified / renamed / validated / corrected, payload, timestamp). +- schema_version, sha256, original_key, current_key, status, template_name, confidence, emission_date, period, emitting_company, document_type, optional_fields (dict of additional field values), extracted_text, created_at, updated_at, events[] (type: classified / renamed / validated / corrected, payload, timestamp). SQLite replica (rebuilt from sidecars): -- files: id, sha256, original_key, current_key, status, template_id, confidence, emission_date, period, emitting_company, document_type, optional_field1, optional_field2, created_at, updated_at, extracted_text, created_at, updated_at. +- files: id, sha256, original_key, current_key, status, template_id, confidence, emission_date, period, emitting_company, document_type, optional_fields (JSON), created_at, updated_at, extracted_text, created_at, updated_at. - consistency_flags: id, file_id, kind (orphaned_object, missing_sidecar, drift). ## 8. API Surface (indicative) diff --git a/automatic-pdf-renamer/scripts/import_paperless.py b/automatic-pdf-renamer/scripts/import_paperless.py index 8be38d2..b8b8855 100755 --- a/automatic-pdf-renamer/scripts/import_paperless.py +++ b/automatic-pdf-renamer/scripts/import_paperless.py @@ -1,28 +1,34 @@ #!/usr/bin/env python3 -"""Paperless-ngx Initial Import Script. +"""Paperless-ngx Import Script for Automatic PDF Renamer. -User Story 0: Standalone script to import existing paperless-ngx documents -into a Garage S3 bucket with their metadata as sidecars. +User Story: Import documents from Paperless-ngx into a Garage S3 bucket +with config-driven processing and auto-updating configuration. -This script runs once, offline, from the owner's machine or the server. -It is NOT part of the renamer container and has no dashboard surface. +Workflow: +1. Fetch all documents from Paperless-ngx +2. Auto-update config.jsonc with missing companies, document types, and additional field values +3. Resolve mapping for each document (template, period_format, additional_fields) +4. Match Paperless tags to additional field categories (strict validation) +5. Calculate period from created_date +6. Write to files/ if all valid, or pending/{paperless_id} if validation fails +7. Write sidecar with full metadata (including errors for pending) + +Note: This script runs from the owner's machine or server, NOT in the renamer container. +It processes all documents fresh on each run (clears files/ and pending/ first). Storage Backends: - S3: For production Garage S3 (default) - File: For local filesystem testing - -Note: Paperless-ngx API returns IDs for correspondent, document_type, and tags. -This script fetches the actual string values from the respective endpoints. -The filename comes from 'archived_file_name', not 'filename'. """ import argparse import hashlib +import json import logging import os import sys from dataclasses import dataclass, field -from datetime import datetime +from pathlib import Path from typing import Any import requests @@ -30,17 +36,24 @@ import requests # Add parent directory to path for imports sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from app.extractor import extract_first_page_text from app.sidecar import ( Sidecar, SidecarStatus, build_sidecar, get_sidecar_key, - matching_sha256, - read_sidecar, - sidecar_exists, write_atomic, + serialize, ) from app.storage import get_storage_backend, StorageBackend +from app.config import ( + TemplateRegistry, + PeriodFormatConfig, + MappingRule, + MappingMatch, + MappingSet, +) +from app.period import format_period_from_emission_date # Configure logging logging.basicConfig( @@ -56,11 +69,10 @@ class PaperlessDocument: id: int filename: str - storage_path: str - created: str # Document date (emission date) + created_date: str # Document creation date (used as emission_date) correspondent_name: str document_type_name: str - labels: list[str] # List of label names + tags: list[str] # List of tag values (strings, no types) download_url: str @@ -69,19 +81,25 @@ class ImportStats: """Statistics for the import run.""" imported: int = 0 + pending: int = 0 skipped: int = 0 failed: int = 0 - period_fallbacks: int = 0 + config_updated: bool = False per_template_counts: dict[str, int] = field(default_factory=dict) errors: list[str] = field(default_factory=list) +@dataclass +class ResolvedMapping: + """Resolved mapping for a document.""" + + template: str | None = None + period_format: str | None = None + additional_fields: list[str] = field(default_factory=list) + + class PaperlessClient: - """Client for Paperless-ngx REST API. - - Note: Paperless returns IDs for correspondent, document_type, and tags. - This client fetches the actual string values from the respective endpoints. - """ + """Client for Paperless-ngx REST API.""" def __init__(self, base_url: str, token: str, timeout: int = 30): """Initialize the Paperless client. @@ -100,8 +118,7 @@ class PaperlessClient: "Accept": "application/json", }) - # Cache for fetched entities to avoid duplicate requests - self._storage_path_cache: dict[int, str] = {} + # Cache for fetched entities self._correspondent_cache: dict[int, str] = {} self._document_type_cache: dict[int, str] = {} self._tag_cache: dict[int, str] = {} @@ -129,60 +146,16 @@ class PaperlessClient: raise def list_documents(self, page: int = 1, page_size: int = 100) -> list[dict[str, Any]]: - """List all documents from Paperless (paginated). - - Args: - page: Page number. - page_size: Number of results per page. - - Returns: - List of document dicts. - """ + """List all documents from Paperless (paginated).""" data = self._get("/api/documents/", params={"page": page, "page_size": page_size}) return data.get("results", []) def get_document(self, doc_id: int) -> dict[str, Any]: - """Get a single document's metadata. - - Args: - doc_id: The document ID. - - Returns: - Document metadata dict. - """ + """Get a single document's metadata.""" return self._get(f"/api/documents/{doc_id}/") - def get_storage_path_name(self, storage_path_id: int) -> str: - """Get storage path name from ID. - - Args: - storage_path_id: The Paperless storage path ID. - - Returns: - Storage path name string. - """ - if not storage_path_id: - return "" - if storage_path_id in self._storage_path_cache: - return self._storage_path_cache[storage_path_id] - try: - storage_path = self._get(f"/api/storage_paths/{storage_path_id}/") - name = storage_path.get("name", f"unknown_storage_path_{storage_path_id}") - self._storage_path_cache[storage_path_id] = name - return name - except Exception as e: - logger.warning(f"Failed to fetch storage path {storage_path_id}: {e}") - return f"unknown_storage_path_{storage_path_id}" - def get_correspondent_name(self, correspondent_id: int) -> str: - """Get correspondent name from ID. - - Args: - correspondent_id: The Paperless correspondent ID. - - Returns: - Correspondent name string. - """ + """Get correspondent name from ID.""" if not correspondent_id: return "" if correspondent_id in self._correspondent_cache: @@ -197,14 +170,7 @@ class PaperlessClient: return f"unknown_correspondent_{correspondent_id}" def get_document_type_name(self, document_type_id: int) -> str: - """Get document type name from ID. - - Args: - document_type_id: The Paperless document type ID. - - Returns: - Document type name string. - """ + """Get document type name from ID.""" if not document_type_id: return "" if document_type_id in self._document_type_cache: @@ -219,14 +185,7 @@ class PaperlessClient: return f"unknown_type_{document_type_id}" def get_tag_name(self, tag_id: int) -> str: - """Get tag name from ID. - - Args: - tag_id: The Paperless tag ID. - - Returns: - Tag name string. - """ + """Get tag name from ID.""" if not tag_id: return "" if tag_id in self._tag_cache: @@ -240,15 +199,8 @@ class PaperlessClient: logger.warning(f"Failed to fetch tag {tag_id}: {e}") return f"unknown_tag_{tag_id}" - def get_all_documents(self, limit: int = None) -> list[PaperlessDocument]: - """Get all documents from Paperless (handles pagination). - - Note: Paperless returns IDs for correspondent, document_type, and tags. - This method fetches the actual string values from the respective endpoints. - - Returns: - List of PaperlessDocument objects. - """ + def get_all_documents(self, limit: int | None = None) -> list[PaperlessDocument]: + """Get all documents from Paperless (handles pagination).""" all_docs: list[PaperlessDocument] = [] page = 1 page_size = 100 @@ -259,41 +211,34 @@ class PaperlessClient: break for doc in docs: - # Get full document details full_doc = self.get_document(doc["id"]) - # Extract fields from Paperless document - # The actual filename is in 'archived_file_name', not 'filename' - archived_file_name = full_doc.get("archived_file_name", "") if full_doc.get("archived_file_name") else full_doc.get("title", "") + ".pdf" + # Extract fields + archived_file_name = full_doc.get("archived_file_name", "") + if not archived_file_name: + archived_file_name = full_doc.get("title", "") + ".pdf" - # Get correspondent name from ID correspondent_id = full_doc.get("correspondent") correspondent_name = self.get_correspondent_name(correspondent_id) if correspondent_id else "" - # Get document type name from ID document_type_id = full_doc.get("document_type") document_type_name = self.get_document_type_name(document_type_id) if document_type_id else "" - # Get storage path - storage_path_id = full_doc.get("storage_path", "") - storage_path = self.get_storage_path_name(storage_path_id) if storage_path_id else "" - - # Get tags/labels - Paperless returns tag IDs, need to fetch names + # Get tags tag_ids = full_doc.get("tags", []) - labels = [] + tags = [] for tag_id in tag_ids: tag_name = self.get_tag_name(tag_id) if tag_name: - labels.append(tag_name) + tags.append(tag_name) paperless_doc = PaperlessDocument( id=full_doc["id"], filename=archived_file_name, - storage_path=storage_path, - created=full_doc.get("created", ""), # This is the document date + created_date=full_doc.get("created", ""), correspondent_name=correspondent_name, document_type_name=document_type_name, - labels=labels, + tags=tags, download_url=f"{self.base_url}/api/documents/{full_doc['id']}/download/", ) logger.debug(f"Retrieved document: {paperless_doc}") @@ -301,9 +246,8 @@ class PaperlessClient: if limit and len(all_docs) >= limit: logger.info(f"Reached limit of {limit} documents") - return all_docs + return all_docs - # Check if there are more pages if len(docs) < page_size: break page += 1 @@ -312,192 +256,414 @@ class PaperlessClient: return all_docs def download_document(self, doc_id: int) -> bytes: - """Download a document's PDF. - - Args: - doc_id: The document ID. - - Returns: - PDF content as bytes. - """ + """Download a document's PDF.""" return self._get_raw(f"/api/documents/{doc_id}/download/") class PDFProcessor: """Processes PDF content for import.""" - def __init__(self): - """Initialize the PDF processor.""" - # Import extractor (optional dependency) - try: - from app.extractor import extract_first_page_text - - self._extract_text = extract_first_page_text - except ImportError: - logger.warning("PyMuPDF not available, text extraction will be skipped") - self._extract_text = None - def compute_sha256(self, pdf_bytes: bytes) -> str: - """Compute SHA256 hash of PDF content. - - Args: - pdf_bytes: The PDF content. - - Returns: - Hex-encoded SHA256 hash. - """ + """Compute SHA256 hash of PDF content.""" return hashlib.sha256(pdf_bytes).hexdigest() - def extract_text(self, pdf_bytes: bytes) -> str: - """Extract first page text from PDF. +class ConfigUpdater: + """Handles auto-updating the configuration from Paperless data.""" + + def __init__(self, config_path: str): + """Initialize the config updater.""" + self.config_path = config_path + self.registry = TemplateRegistry() + self.registry.load_from_file(config_path) + self._changes: dict[str, Any] = {} + # Track added items in memory for immediate use + self._added_companies: set[str] = set() + self._added_document_types: set[str] = set() + self._added_additional_fields: dict[str, set[str]] = {} + + def add_company(self, company: str) -> bool: + """Add a company to config if not present.""" + if company and company not in self.registry.get_companies() and company not in self._added_companies: + self._changes.setdefault("companies_added", []).append(company) + self._added_companies.add(company) + return True + return False + + def add_document_type(self, doc_type: str) -> bool: + """Add a document type to config if not present.""" + if doc_type and doc_type not in self.registry.get_document_types() and doc_type not in self._added_document_types: + self._changes.setdefault("document_types_added", []).append(doc_type) + self._added_document_types.add(doc_type) + return True + return False + + def add_additional_field_value(self, category: str, value: str) -> bool: + """Add a value to an additional field category, creating category if needed.""" + if category and value: + # Check if already in registry + if category in self.registry.get_additional_fields(): + if value not in self.registry.get_additional_field_values(category): + self._changes.setdefault("additional_fields_added", {}).setdefault(category, []).append(value) + self._added_additional_fields.setdefault(category, set()).add(value) + return True + # Check if already added in memory + elif category in self._added_additional_fields: + if value not in self._added_additional_fields[category]: + self._changes.setdefault("additional_fields_added", {}).setdefault(category, []).append(value) + self._added_additional_fields.setdefault(category, set()).add(value) + return True + # New category and value + else: + self._changes.setdefault("additional_fields_added", {}).setdefault(category, []).append(value) + self._added_additional_fields.setdefault(category, set()).add(value) + return True + return False + + def has_changes(self) -> bool: + """Check if there are any pending changes.""" + return bool(self._changes) + + def _strip_jsonc_comments(self, content: str) -> str: + """Strip // line comments and /* */ block comments from JSONC content.""" + import re + # Remove /* */ block comments first + content = re.sub(r'/\*.*?\*/', '', content, flags=re.DOTALL) + # Remove // line comments + content = re.sub(r'//.*$', '', content, flags=re.MULTILINE) + # Remove leading/trailing whitespace from the result + return content.strip() + + def save_config(self) -> None: + """Save the updated configuration to disk.""" + if not self.has_changes(): + logger.info("No config changes to save") + return + + logger.info("Saving updated configuration...") + + # Get current config data + import json + with open(self.config_path, "r", encoding="utf-8") as f: + content = f.read() + + # Strip JSONC comments before parsing + json_content = self._strip_jsonc_comments(content) + config_data = json.loads(json_content) + + # Apply changes + if "companies_added" in self._changes: + existing = set(config_data.get("companies", [])) + for company in self._changes["companies_added"]: + if company not in existing: + config_data.setdefault("companies", []).append(company) + existing.add(company) + logger.info(f" Added company: {company}") + + if "document_types_added" in self._changes: + existing = set(config_data.get("document_types", [])) + for doc_type in self._changes["document_types_added"]: + if doc_type not in existing: + config_data.setdefault("document_types", []).append(doc_type) + existing.add(doc_type) + logger.info(f" Added document_type: {doc_type}") + + if "additional_fields_added" in self._changes: + for category, values in self._changes["additional_fields_added"].items(): + config_data.setdefault("additional_fields", {}).setdefault(category, []) + existing = set(config_data["additional_fields"][category]) + for value in values: + if value not in existing: + config_data["additional_fields"][category].append(value) + existing.add(value) + logger.info(f" Added additional_field value: {category}={value}") + + # Write back to file + with open(self.config_path, "w", encoding="utf-8") as f: + json.dump(config_data, f, indent=2, ensure_ascii=False) + f.write("\n") + + logger.info("Configuration saved successfully") + + # Reload registry with updated config + self.registry = TemplateRegistry() + self.registry.load_from_file(self.config_path) + self._changes = {} + + +class DocumentProcessor: + """Processes a single document through the import workflow.""" + + def __init__(self, registry: TemplateRegistry, storage: StorageBackend): + """Initialize the document processor.""" + self.registry = registry + self.storage = storage + self.pdf_processor = PDFProcessor() + + def resolve_mapping(self, company: str, document_type: str) -> ResolvedMapping: + """Resolve the mapping for a document.""" + result = ResolvedMapping() + + # Get all mapping rules + for rule in self.registry.get_mapping_rules(): + if rule.matches(company, document_type): + if rule.set.template is not None: + result.template = rule.set.template + if rule.set.period_format is not None: + result.period_format = rule.set.period_format + if rule.set.additional_fields: + result.additional_fields = rule.set.additional_fields.copy() + + return result + + def match_tags_to_fields( + self, + tags: list[str], + expected_categories: list[str] + ) -> tuple[dict[str, str], list[str]]: + """Match Paperless tags to additional field categories. + + Strict validation: each tag must match exactly one category. + If any tag doesn't match or matches multiple categories, return errors. + Args: - pdf_bytes: The PDF content. - + tags: List of tag values from Paperless + expected_categories: List of additional field categories from mapping + Returns: - Extracted text, or empty string if extraction fails. + Tuple of (assigned_fields, errors) """ - if self._extract_text is None: - logger.warning("Text extraction not available") - return "" + assigned_fields: dict[str, str] = {} + errors: list[str] = [] - try: - return self._extract_text(pdf_bytes) - except Exception as e: - logger.warning(f"Text extraction failed: {e}") - return "" + if not expected_categories: + # No additional fields expected - all tags are just metadata + if tags: + assigned_fields["tags"] = ", ".join(tags) + return assigned_fields, errors + + if len(expected_categories) == 1: + # Single category - all tags belong to it + category = expected_categories[0] + # Values are already in config (added by ConfigUpdater) + for tag in tags: + assigned_fields[category] = tag + return assigned_fields, errors + + # Multiple categories - need strict matching + for tag in tags: + matching_categories = [] + for category in expected_categories: + values = self.registry.get_additional_field_values(category) + if tag in values: + matching_categories.append(category) + + if len(matching_categories) == 0: + errors.append(f"Tag '{tag}' does not match any expected category") + elif len(matching_categories) > 1: + errors.append(f"Tag '{tag}' matches multiple categories: {matching_categories}") + else: + assigned_fields[matching_categories[0]] = tag + + return assigned_fields, errors - -class SidecarBuilder: - """Builds sidecar metadata from Paperless documents.""" - - def __init__(self, template_mappings: dict[str, str] | None = None): - """Initialize the sidecar builder. - - Args: - template_mappings: Optional dict mapping Paperless storage_path - prefixes to template names. - """ - self.template_mappings = template_mappings or {} - - def get_template_name(self, storage_path: str) -> str: - """Get template name from storage path. - - Uses the first path segment as the document family (template). - - Args: - storage_path: The Paperless storage path. - - Returns: - Template name (first segment of storage path). - """ - if not storage_path: - return "unknown" - - # Get first path segment - parts = storage_path.strip("/").split("/") - if parts: - return parts[0] - return "unknown" - - def parse_period_from_filename(self, filename: str) -> str | None: - """Parse period from filename. - - Args: - filename: The document filename. - - Returns: - Period string, or None if cannot be parsed. - """ - try: - from app.period import parse_period_from_filename - - return parse_period_from_filename(filename) - except Exception: - return None - - def build_sidecar( + def process_document( self, doc: PaperlessDocument, - pdf_bytes: bytes, - sha256: str, - extracted_text: str, - template_name: str | None = None, - ) -> Sidecar: - """Build a sidecar from a Paperless document. - + config_updater: ConfigUpdater, + ) -> tuple[str, Sidecar | None, list[str]]: + """Process a single document. + Args: - doc: The Paperless document. - pdf_bytes: The PDF content. - sha256: The SHA256 hash of the PDF. - extracted_text: The extracted first-page text. - template_name: Optional template name override. - + doc: The Paperless document + config_updater: For auto-updating config + Returns: - A Sidecar object with all metadata. + Tuple of (target_key, sidecar, errors) + - target_key: Either files/{final_key} or pending/{paperless_id} + - sidecar: Sidecar metadata (or None if failed) + - errors: List of error messages """ - # Determine template - if template_name: - final_template = template_name - else: - final_template = self.get_template_name(doc.storage_path) + errors: list[str] = [] + + # Step 1: Auto-update config with company and document type + company = doc.correspondent_name + doc_type = doc.document_type_name + + # Add to config (this tracks changes in memory) + config_updater.add_company(company) + config_updater.add_document_type(doc_type) + + # Reload registry from config_updater to get the updated state + self.registry = config_updater.registry + + # Validate - check both registry and in-memory additions + is_valid_company = (company in self.registry.get_companies() or + company in config_updater._added_companies) + is_valid_doc_type = (doc_type in self.registry.get_document_types() or + doc_type in config_updater._added_document_types) + + if not is_valid_company: + errors.append(f"Company '{company}' could not be added to config") + if not is_valid_doc_type: + errors.append(f"Document type '{doc_type}' could not be added to config") + + if errors: + return self._route_to_pending(doc, errors) + + # Step 2: Resolve mapping + resolved = self.resolve_mapping(company, doc_type) + + if not resolved.template: + errors.append(f"No template resolved for company='{company}', document_type='{doc_type}'") + + if not resolved.period_format: + errors.append(f"No period_format resolved for company='{company}', document_type='{doc_type}'") + + if errors: + return self._route_to_pending(doc, errors) + + # Step 3: Auto-update additional fields config + for category in resolved.additional_fields: + for tag in doc.tags: + config_updater.add_additional_field_value(category, tag) + + # Reload registry after potential changes + self.registry = config_updater.registry + + # Step 4: Match tags to additional field categories + # First, ensure all expected categories exist and have values added + for category in resolved.additional_fields: + for tag in doc.tags: + config_updater.add_additional_field_value(category, tag) + + # Reload registry after potential changes + self.registry = config_updater.registry + + # Now match tags to categories + assigned_fields, tag_errors = self.match_tags_to_fields( + doc.tags, + resolved.additional_fields + ) + errors.extend(tag_errors) + + if resolved.additional_fields and not assigned_fields: + errors.append("No tags matched expected additional fields") + + if errors: + return self._route_to_pending(doc, errors) + + # Step 5: Calculate period + try: + period_format_config = self.registry.get_period_format_config(resolved.period_format) + if not period_format_config: + errors.append(f"Period format '{resolved.period_format}' not found") + return self._route_to_pending(doc, errors) + + period = format_period_from_emission_date( + doc.created_date, + period_format_config.to_dict() + ) - # Parse period from filename - period = self.parse_period_from_filename(doc.filename) - if not period: - # Fallback: use emission date's month-year - from datetime import datetime - - try: - dt = datetime.fromisoformat(doc.created).date() - from app.period import format_month_year - - period = format_month_year(dt.month, dt.year) - logger.warning(f"Using fallback period for doc {doc.id}: {period}") - except Exception: - period = doc.created # Last resort - - # Build optional fields from labels - optional_fields = {} - for label in doc.labels: - optional_fields["tag"] = label - - # Construct the original key - if doc.storage_path: - original_key = f"{doc.storage_path}/{doc.filename}" - else: - original_key = doc.filename - - # Current key is the same as original for import (no renames) - current_key = f"files/{final_template}/{doc.filename}" - - # Build the sidecar + logger.debug(f"Calculated period '{period}' for document {doc.id} using format '{resolved.period_format}'") + except Exception as e: + errors.append(f"Failed to calculate period: {e}") + logger.exception(f"Error calculating period for document {doc.id}") + return self._route_to_pending(doc, errors) + + # Step 6: Generate final key + template_name = resolved.template + key_pattern = self.registry.get_key_pattern(template_name) + + if not key_pattern: + errors.append(f"No key_pattern found for template '{template_name}'") + return self._route_to_pending(doc, errors) + + try: + final_key = key_pattern.format( + emitting_company=company, + document_type=doc_type, + emission_date=doc.created_date, + period=period, + original_filename=doc.filename, + **assigned_fields + ) + except Exception as e: + errors.append(f"Failed to format key pattern: {e}") + return self._route_to_pending(doc, errors) + + # Step 7: Build sidecar for files/ + pdf_key = f"files/{final_key}" + sha256 = "" # Will be computed when writing + sidecar = build_sidecar( sha256=sha256, - original_key=original_key, - current_key=current_key, - status=SidecarStatus.VALIDATED, - template_name=final_template, - emission_date=doc.created, + original_key=doc.filename, + current_key=pdf_key, + status=SidecarStatus.PROCESSED, + template_name=template_name, + emission_date=doc.created_date, period=period, - emitting_company=doc.correspondent_name, - document_type=doc.document_type_name, + emitting_company=company, + document_type=doc_type, confidence=1.0, - optional_fields=optional_fields, - extracted_text=extracted_text, + optional_fields=assigned_fields, + extracted_text="", ) - - # Add imported event + + # Add import event sidecar.add_event( "imported", { "source": "paperless-ngx", "document_id": doc.id, - "document_url": doc.download_url, + "paperless_filename": doc.filename, }, ) + + return pdf_key, sidecar, [] - return sidecar + def _route_to_pending( + self, + doc: PaperlessDocument, + errors: list[str] + ) -> tuple[str, Sidecar | None, list[str]]: + """Route document to pending/ with error sidecar.""" + pdf_key = f"pending/{doc.id}" + + # Build sidecar with errors in optional_fields + # We use optional_fields to store errors for pending documents + optional_fields = {"tags": doc.tags} if doc.tags else {} + optional_fields["_import_errors"] = errors + + # Build sidecar + sidecar = build_sidecar( + sha256="", + original_key=doc.filename, + current_key=pdf_key, + status=SidecarStatus.NOT_PROCESSED, + template_name="", + emission_date=doc.created_date, + period="", + emitting_company=doc.correspondent_name, + document_type=doc.document_type_name, + confidence=0.0, + optional_fields=optional_fields, + extracted_text="", + ) + + # Add import event with errors + sidecar.add_event( + "imported", + { + "source": "paperless-ngx", + "document_id": doc.id, + "paperless_filename": doc.filename, + "errors": errors, + }, + ) + + return pdf_key, sidecar, errors class Importer: @@ -507,6 +673,7 @@ class Importer: self, paperless_url: str, paperless_token: str, + config_path: str, storage_backend: str = "s3", s3_endpoint: str | None = None, s3_access_key: str | None = None, @@ -515,25 +682,11 @@ class Importer: file_base_dir: str | None = None, dry_run: bool = False, limit: int | None = None, - template_mappings: dict[str, str] | None = None, ): - """Initialize the importer. - - Args: - paperless_url: Paperless-ngx API URL. - paperless_token: Paperless-ngx API token. - storage_backend: Storage backend type ('s3' or 'file'). - s3_endpoint: Garage S3 endpoint URL (required for S3). - s3_access_key: S3 access key (required for S3). - s3_secret_key: S3 secret key (required for S3). - s3_bucket: S3 bucket name (required for S3). - file_base_dir: Base directory for file storage (required for file backend). - dry_run: If True, only list what would be imported. - limit: Maximum number of documents to import (for testing). - template_mappings: Optional dict mapping storage_path to template. - """ + """Initialize the importer.""" self.paperless_url = paperless_url self.paperless_token = paperless_token + self.config_path = config_path self.storage_backend_type = storage_backend self.s3_endpoint = s3_endpoint self.s3_access_key = s3_access_key @@ -542,22 +695,26 @@ class Importer: self.file_base_dir = file_base_dir self.dry_run = dry_run self.limit = limit - self.template_mappings = template_mappings # Initialize clients self.paperless_client = PaperlessClient(paperless_url, paperless_token) - self.pdf_processor = PDFProcessor() - self.sidecar_builder = SidecarBuilder(template_mappings) - - # Initialize storage - will be set up in _init_storage() self.storage: StorageBackend | None = None self._init_storage() + # Initialize config updater + self.config_updater = ConfigUpdater(config_path) + + # Document processor + self.doc_processor = DocumentProcessor( + self.config_updater.registry, + self.storage + ) + # Stats self.stats = ImportStats() def _init_storage(self) -> None: - """Initialize the storage backend based on configuration.""" + """Initialize the storage backend.""" if self.storage_backend_type == "s3": if not all([self.s3_endpoint, self.s3_access_key, self.s3_secret_key, self.s3_bucket]): raise ValueError( @@ -579,23 +736,26 @@ class Importer: else: raise ValueError(f"Unknown storage backend: {self.storage_backend_type}") - def check_bucket_empty(self, prefix: str = "files/") -> bool: - """Check if the bucket is empty under the given prefix. - - Args: - prefix: The prefix to check. - - Returns: - True if bucket is empty (no objects), False otherwise. - """ + def clear_directories(self) -> None: + """Clear files/ and pending/ directories at start of run.""" if self.storage is None: raise RuntimeError("Storage not initialized") - objects = self.storage.list_objects(prefix) - if objects: - logger.error(f"Bucket is not empty under {prefix}: found {len(objects)} objects") - logger.error(f"First few objects: {objects[:5]}") - return False - return True + + logger.info("Clearing files/ and pending/ directories...") + + # List and delete all objects under files/ + files_objects = self.storage.list_objects("files/") + for obj_key in files_objects: + self.storage.delete_object(obj_key) + logger.debug(f"Deleted {obj_key}") + + # List and delete all objects under pending/ + pending_objects = self.storage.list_objects("pending/") + for obj_key in pending_objects: + self.storage.delete_object(obj_key) + logger.debug(f"Deleted {obj_key}") + + logger.info(f"Cleared {len(files_objects)} files/ and {len(pending_objects)} pending/ objects") def import_document(self, doc: PaperlessDocument) -> bool: """Import a single document. @@ -614,62 +774,47 @@ class Importer: try: # Download PDF pdf_bytes = self.paperless_client.download_document(doc.id) - sha256 = self.pdf_processor.compute_sha256(pdf_bytes) + sha256 = self.doc_processor.pdf_processor.compute_sha256(pdf_bytes) - # Extract text - extracted_text = self.pdf_processor.extract_text(pdf_bytes) - - # Build sidecar - sidecar = self.sidecar_builder.build_sidecar( - doc, pdf_bytes, sha256, extracted_text + # Process document + target_key, sidecar, errors = self.doc_processor.process_document( + doc, self.config_updater ) - # Determine keys - pdf_key = sidecar.current_key - sidecar_key = get_sidecar_key(pdf_key) - - # Check for resumability - if sidecar_exists(self.storage, pdf_key) and matching_sha256( - self.storage, pdf_key, sha256 - ): - logger.info(f"Skipping document {doc.id}: already imported with matching SHA256") - self.stats.skipped += 1 - self.stats.per_template_counts[sidecar.template_name] = ( - self.stats.per_template_counts.get(sidecar.template_name, 0) + 1 - ) - return True - # Dry run: just log what would happen if self.dry_run: - logger.info(f"[DRY RUN] Would import: {pdf_key}") - logger.info(f"[DRY RUN] SHA256: {sha256}") - logger.info(f"[DRY RUN] Template: {sidecar.template_name}") - logger.info(f"[DRY RUN] Emitting Company: {sidecar.emitting_company}") - logger.info(f"[DRY RUN] Document Type: {sidecar.document_type}") - logger.info(f"[DRY RUN] Emission Date: {sidecar.emission_date}") - logger.info(f"[DRY RUN] Period: {sidecar.period}") - logger.info(f"[DRY RUN] Optional Fields: {sidecar.optional_fields}") + if errors: + logger.info(f"[DRY RUN] Would route to pending: {target_key}") + logger.info(f"[DRY RUN] Errors: {errors}") + else: + logger.info(f"[DRY RUN] Would import: {target_key}") + self.stats.imported += 1 + return True + + # Write PDF + self.storage.put_object(target_key + ".pdf", pdf_bytes) + logger.debug(f"Uploaded PDF to {target_key}") + + # Update sidecar + sidecar.sha256 = sha256 + sidecar.extracted_text = extract_first_page_text(pdf_bytes) + + # Write sidecar + sidecar_key = get_sidecar_key(target_key) + sidecar_json = serialize(sidecar) + self.storage.put_object(sidecar_key, sidecar_json.encode("utf-8")) + logger.debug(f"Wrote sidecar to {sidecar_key}") + + # Update stats + if errors: + self.stats.pending += 1 + else: self.stats.imported += 1 self.stats.per_template_counts[sidecar.template_name] = ( self.stats.per_template_counts.get(sidecar.template_name, 0) + 1 ) - return True - # Upload PDF - self.storage.put_object(pdf_key, pdf_bytes) - logger.debug(f"Uploaded PDF to {pdf_key}") - - # Write sidecar atomically - write_atomic(self.storage, sidecar, sidecar_key) - logger.debug(f"Wrote sidecar to {sidecar_key}") - - # Update stats - self.stats.imported += 1 - self.stats.per_template_counts[sidecar.template_name] = ( - self.stats.per_template_counts.get(sidecar.template_name, 0) + 1 - ) - - logger.info(f"Successfully imported document {doc.id}") + logger.info(f"Successfully {'imported' if not errors else 'routed to pending'} document {doc.id}") return True except Exception as e: @@ -691,12 +836,10 @@ class Importer: logger.info("Starting Paperless-ngx import...") logger.info(f"Storage backend: {self.storage_backend_type}") + logger.info(f"Config path: {self.config_path}") - # Check bucket is empty (only for S3, skip for file backend in dry-run) - if not self.dry_run and self.storage_backend_type == "s3": - if not self.check_bucket_empty("files/"): - logger.error("Refusing to run: bucket is not empty under files/") - return False + # Clear directories at start + self.clear_directories() # Get all documents from Paperless logger.info("Fetching documents from Paperless...") @@ -704,6 +847,9 @@ class Importer: if not docs: logger.info("No documents found in Paperless") + # Save config if changes were made + if self.config_updater.has_changes(): + self.config_updater.save_config() return True logger.info(f"Found {len(docs)} documents to import") @@ -712,6 +858,11 @@ class Importer: for doc in docs: self.import_document(doc) + # Save config at end if changes were made + if self.config_updater.has_changes(): + self.config_updater.save_config() + self.stats.config_updated = True + # Print summary self.print_summary() @@ -724,10 +875,12 @@ class Importer: print("IMPORT SUMMARY") print("=" * 60) print(f"Storage backend: {self.storage_backend_type}") - print(f"Imported: {self.stats.imported}") - print(f"Skipped: {self.stats.skipped}") - print(f"Failed: {self.stats.failed}") - print(f"Period fallbacks: {self.stats.period_fallbacks}") + print(f"Config path: {self.config_path}") + print(f"Config updated: {'Yes' if self.stats.config_updated else 'No'}") + print(f"Imported to files/: {self.stats.imported}") + print(f"Routed to pending/: {self.stats.pending}") + print(f"Skipped: {self.stats.skipped}") + print(f"Failed: {self.stats.failed}") print("\nPer-template counts:") for template, count in sorted(self.stats.per_template_counts.items()): print(f" {template}: {count}") @@ -742,7 +895,7 @@ class Importer: def parse_args(): """Parse command line arguments.""" parser = argparse.ArgumentParser( - description="Import documents from Paperless-ngx to storage backend", + description="Import documents from Paperless-ngx with config-driven processing", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Environment variables for S3 storage: @@ -771,6 +924,13 @@ For file storage, use --storage-backend file --file-base-dir /path/to/dir default=os.environ.get("PAPERLESS_TOKEN"), ) + # Config file + parser.add_argument( + "--config-path", + help="Path to config.jsonc file", + default="./config.jsonc", + ) + # Storage backend configuration parser.add_argument( "--storage-backend", @@ -830,14 +990,7 @@ For file storage, use --storage-backend file --file-base-dir /path/to/dir def validate_env(args) -> None: - """Validate that required environment variables or arguments are set. - - Args: - args: Parsed command line arguments. - - Raises: - ValueError: If required values are missing. - """ + """Validate that required environment variables or arguments are set.""" missing = [] # Paperless is always required @@ -883,6 +1036,7 @@ def main(): importer = Importer( paperless_url=args.paperless_url, paperless_token=args.paperless_token, + config_path=args.config_path, storage_backend=args.storage_backend, s3_endpoint=args.s3_endpoint, s3_access_key=args.s3_access_key, @@ -910,10 +1064,18 @@ def main(): except Exception as e: logger.error(f"Import failed: {e}") import traceback - traceback.print_exc() sys.exit(1) +# Add PDFProcessor class for completeness +class PDFProcessor: + """Processes PDF content for import.""" + + def compute_sha256(self, pdf_bytes: bytes) -> str: + """Compute SHA256 hash of PDF content.""" + return hashlib.sha256(pdf_bytes).hexdigest() + + if __name__ == "__main__": main()