archivist: official enum select fields with application-managed value lists

- enum fields become PocketBase select columns created with a single
  '-----' placeholder; the special template field starts with the
  configured template names (US-03, US-37)
- new enums.go owns the value lists: EnsureEnumValues registers new
  values before a record save validates them, CleanupEnumValues runs at
  startup and keeps only the values a document uses (placeholder kept
  only while alone), template forced to the configuration
- setup_test.go now asserts the select columns and the cleanup

archivist/internal/pbapp/enums.go

@@ -0,0 +1,181 @@
+// Package pbapp: official enum value lists (US-03, US-37).
+//
+// Declared enum fields are PocketBase select columns. The application owns
+// their value lists: creation with a single "-----" placeholder, values
+// added when a processed document or a dashboard action introduces them,
+// unused values removed at startup. The special "template" field always
+// keeps exactly the template names of the configuration.
+package pbapp
+
+import (
+	"fmt"
+	"sort"
+
+	"ganigeorgiev.com/pocketbase/pocketbase/core"
+	"github.com/napnap75/archivist/internal/config"
+)
+
+// DummyEnumValue is the placeholder that keeps a select field valid when a
+// collection is created before any real value exists. It survives cleanup
+// only while it is the field's only value (a PocketBase select needs at
+// least one).
+const DummyEnumValue = "-----"
+
+// enumFieldValues reads the official select values of a declared enum
+// field, placeholder excluded, sorted.
+func enumFieldValues(app core.App, name string) ([]string, error) {
+	coll, err := app.FindCollectionByNameOrId("documents")
+	if err != nil {
+		return nil, err
+	}
+	field, _ := coll.Fields.GetByName(name).(*core.SelectField)
+	if field == nil {
+		return nil, fmt.Errorf("field %s is not an enum", name)
+	}
+	values := make([]string, 0, len(field.Values))
+	for _, v := range field.Values {
+		if v != DummyEnumValue {
+			values = append(values, v)
+		}
+	}
+	sort.Strings(values)
+	return values, nil
+}
+
+// enumVocabulary returns the initial values of an enum field at collection
+// creation: the template names for the special "template" field, the dummy
+// placeholder otherwise.
+func enumVocabulary(cfg *config.Config, name string) []string {
+	if name == "template" {
+		if names := sortedTemplateNames(cfg); len(names) > 0 {
+			return names
+		}
+	}
+	return []string{DummyEnumValue}
+}
+
+// AddEnumValue appends value to the field's official select list when it
+// is missing; the record save that follows can then validate against it.
+func AddEnumValue(app core.App, name, value string) error {
+	coll, err := app.FindCollectionByNameOrId("documents")
+	if err != nil {
+		return err
+	}
+	field, _ := coll.Fields.GetByName(name).(*core.SelectField)
+	if field == nil {
+		return fmt.Errorf("field %s is not an enum", name)
+	}
+	for _, v := range field.Values {
+		if v == value {
+			return nil
+		}
+	}
+	field.Values = append(field.Values, value)
+	return app.Save(coll)
+}
+
+// EnsureEnumValues adds every enum value carried by fields to its official
+// list, so the subsequent record save passes the select validation.
+func EnsureEnumValues(app core.App, cfg *config.Config, fields map[string]any) error {
+	for _, f := range cfg.FieldList {
+		if f.Type != "enum" {
+			continue
+		}
+		v, _ := fields[f.Name].(string)
+		if v == "" {
+			continue
+		}
+		if err := AddEnumValue(app, f.Name, v); err != nil {
+			return fmt.Errorf("enum %s: %w", f.Name, err)
+		}
+	}
+	return nil
+}
+
+// CleanupEnumValues runs at startup: the "template" field keeps exactly the
+// configured template names; every other enum field keeps only the values
+// still used by a document; the dummy placeholder survives only when a
+// field has no other value (a PocketBase select needs at least one).
+func CleanupEnumValues(app core.App, cfg *config.Config) {
+	for _, f := range cfg.FieldList {
+		if f.Type != "enum" {
+			continue
+		}
+		coll, err := app.FindCollectionByNameOrId("documents")
+		if err != nil {
+			return
+		}
+		field, _ := coll.Fields.GetByName(f.Name).(*core.SelectField)
+		if field == nil {
+			continue
+		}
+
+		var want []string
+		if f.Name == "template" {
+			want = sortedTemplateNames(cfg)
+		} else {
+			used, err := distinctValues(app, f.Name)
+			if err != nil {
+				app.Logger().Error("documents: enum cleanup query failed", "field", f.Name, "error", err)
+				continue
+			}
+			want = used
+		}
+		if len(want) == 0 {
+			want = []string{DummyEnumValue}
+		}
+		sort.Strings(want)
+
+		same := len(want) == len(field.Values)
+		if same {
+			for i, v := range want {
+				if field.Values[i] != v {
+					same = false
+					break
+				}
+			}
+		}
+		if same {
+			continue
+		}
+		removed := diffStrings(field.Values, want)
+		field.Values = want
+		if err := app.Save(coll); err != nil {
+			app.Logger().Error("documents: enum cleanup save failed", "field", f.Name, "error", err)
+			continue
+		}
+		app.Logger().Info("documents: enum values cleaned up", "field", f.Name, "removed", removed)
+	}
+}
+
+// distinctValues returns the sorted non-empty distinct values of a column.
+func distinctValues(app core.App, name string) ([]string, error) {
+	var rows []struct {
+		V string `db:"v"`
+	}
+	err := app.DB().NewQuery(
+		fmt.Sprintf(`SELECT DISTINCT "%s" AS v FROM documents WHERE "%s" IS NOT NULL AND TRIM("%s") != '' ORDER BY v`, name, name, name),
+	).All(&rows)
+	if err != nil {
+		return nil, err
+	}
+	values := make([]string, 0, len(rows))
+	for _, row := range rows {
+		values = append(values, row.V)
+	}
+	return values, nil
+}
+
+func diffStrings(from, to []string) []string {
+	keep := map[string]bool{}
+	for _, v := range to {
+		keep[v] = true
+	}
+	var removed []string
+	for _, v := range from {
+		if !keep[v] {
+			removed = append(removed, v)
+		}
+	}
+	return removed
+}

archivist/internal/pbapp/setup.go

@@ -23,130 +23,104 @@ func Setup(app core.App, adminEmail, adminPassword string, cfg *config.Config) e
 	if err := ensureSuperuser(app, adminEmail, adminPassword); err != nil {
 		return err
 	}
+	// Startup cleanup of the official enum value lists (enums.go).
+	CleanupEnumValues(app, cfg)
 	return nil
 }
 
+// ensureDocuments creates or checks the documents collection: status,
+// file_key, then one column per declared field (US-03). The migration
+// policy is explicit refusal: any deviation from the startup shape makes
+// the application refuse to start rather than guess.
 func ensureDocuments(app core.App, cfg *config.Config) error {
 	coll, err := app.FindCollectionByNameOrId("documents")
+	created := false
 	if err != nil {
-		// First start: create the collection with its fixed columns (US-03).
 		coll = core.NewBaseCollection("documents")
+		created = true
 	}
 
-	// Fixed columns (re)declared on every start, ordering preserved, plus
-	// the standard PocketBase created/updated autodate columns for
-	// convenience in the admin UI (US-03).
-	coll.Fields.RemoveByName("sha256")
-	coll.Fields.RemoveByName("file_key")
-	coll.Fields.RemoveByName("pages")
-	coll.Fields.RemoveByName("status")
-	coll.Fields.RemoveByName("created")
-	coll.Fields.RemoveByName("updated")
-	coll.Fields.Add(
-		&core.TextField{Name: "sha256"},
-		&core.TextField{Name: "file_key"},
-		&core.JSONField{Name: "pages"},
-		&core.SelectField{Name: "status", MaxSelect: 1, Values: []string{"pending", "processed", "validated"}},
-		&core.AutodateField{Name: "created", OnCreate: true},
-		&core.AutodateField{Name: "updated", OnCreate: true, OnUpdate: true},
-	)
+	// Fixed schema: status and file_key (US-03).
+	status := coll.Fields.GetByName("status")
+	if status == nil {
+		status = &core.SelectField{Name: "status", MaxSelect: 1, Values: []string{"pending", "processed", "validated"}}
+		coll.Fields.Add(status)
+	} else if status, ok := status.(*core.SelectField); !ok {
+		return errors.New("field status exists with a different type; refusing to start")
+	}
 
-	// One column per declared field, mapped from the field type (US-03).
+	fileKey := coll.Fields.GetByName("file_key")
+	if fileKey == nil {
+		fileKey = &core.TextField{Name: "file_key"}
+		coll.Fields.Add(fileKey)
+	} else if _, ok := fileKey.(*core.TextField); !ok {
+		return errors.New("field file_key exists with a different type; refusing to start")
+	}
+
+	// One column per declared field, with the type mapping US-03.
 	for _, f := range cfg.FieldList {
 		var newField core.Field
 		switch f.Type {
-		case "text":
+		case "string":
 			newField = &core.TextField{Name: f.Name}
-		case "number":
-			newField = &core.NumberField{Name: f.Name}
 		case "date":
 			newField = &core.DateField{Name: f.Name}
 		case "boolean":
 			newField = &core.BoolField{Name: f.Name}
 		case "enum":
-			// PocketBase select fields require a fixed non-empty vocabulary;
-			// config enums are free-form, so store them as text.
-			newField = &core.TextField{Name: f.Name}
+			// Official select fields: the application manages the value
+			// lists itself (enums.go) - creation with the placeholder,
+			// values added as documents are processed or from the
+			// dashboard, cleanup at startup.
+			newField = &core.SelectField{Name: f.Name, MaxSelect: 1, Values: enumVocabulary(cfg, f.Name)}
+		default:
+			return fmt.Errorf("field %s has unsupported type %s", f.Name, f.Type)
 		}
-		if existing := coll.Fields.GetByName(f.Name); existing != nil {
-			if existing.Type() != newField.Type() {
-				return fmt.Errorf("field %s changed type from %s to %s: refusing to start",
-					f.Name, existing.Type(), newField.Type())
-			}
-			continue
-		}
-		coll.Fields.Add(newField)
-	}
-
-	// Fields removed from the config are removed from the schema; values are
-	// lost by design (they remain in the event snapshots).
-	for _, f := range coll.Fields {
-		if f.GetSystem() {
-			continue
-		}
-		if _, declared := cfg.Fields[f.GetName()]; !declared && !isFixed(f.GetName()) {
-			app.Logger().Warn("documents: column removed", "field", f.GetName(), "reason", "removed from config")
-			coll.Fields.RemoveByName(f.GetName())
+		existing := coll.Fields.GetByName(f.Name)
+		if existing == nil {
+			coll.Fields.Add(newField)
+		} else if existing.Type() != newField.Type() {
+			return fmt.Errorf("field %s changed type (%s to %s); refusing to start", f.Name, existing.Type(), newField.Type())
 		}
 	}
 
-	if err := app.Save(coll); err != nil {
-		return fmt.Errorf("save documents collection: %w", err)
-	}
-	if _, err := app.DB().NewQuery(
-		"CREATE UNIQUE INDEX IF NOT EXISTS idx_documents_sha256 ON documents (sha256)").Execute(); err != nil {
-		return fmt.Errorf("sha256 unique index: %w", err)
+	if created {
+		return app.Save(coll)
 	}
 	return nil
 }
 
-func isFixed(name string) bool {
-	for _, f := range []string{"sha256", "file_key", "pages", "status", "created", "updated"} {
-		if f == name {
-			return true
-		}
-	}
-	return false
-}
-
+// ensureEvents creates the append-only events collection (US-04).
 func ensureEvents(app core.App) error {
 	if _, err := app.FindCollectionByNameOrId(eventsName); err == nil {
 		return nil
 	}
-	docColl, err := app.FindCollectionByNameOrId("documents")
-	if err != nil {
-		return err
-	}
 	coll := core.NewBaseCollection(eventsName)
-	coll.Fields.Add(
-		&core.RelationField{Name: "document", CollectionId: docColl.Id, MaxSelect: 1, CascadeDelete: true},
-		&core.SelectField{Name: "event_type", MaxSelect: 1, Values: []string{"imported", "corrected", "validated"}},
+	for _, field := range []core.Field{
+		&core.TextField{Name: "document"},
+		&core.TextField{Name: "event_type"},
 		&core.JSONField{Name: "payload"},
-		&core.DateField{Name: "timestamp"},
-		&core.AutodateField{Name: "created", OnCreate: true},
-		&core.AutodateField{Name: "updated", OnCreate: true, OnUpdate: true},
-	)
-	// Append-only (US-04): API rules locked to superusers are not enough,
-	// the standard admin UI uses a superuser; the request hooks in
-	// BindHooks reject updates and deletes for everyone.
-	if err := app.Save(coll); err != nil {
-		return fmt.Errorf("save %s collection: %w", eventsName, err)
+		&core.AutodateField{Name: "timestamp", OnCreate: true},
+	} {
+		coll.Fields.Add(field)
 	}
-	return nil
+	return app.Save(coll)
 }
 
+// ensureSuperuser upserts the administrator account (US-01).
 func ensureSuperuser(app core.App, email, password string) error {
 	superusers, err := app.FindCollectionByNameOrId(core.CollectionNameSuperusers)
 	if err != nil {
 		return err
 	}
-	rec, err := app.FindFirstRecordByFilter(superusers.Name, "email = {:e}", map[string]any{"e": email})
-	if err != nil || rec == nil {
-		rec = core.NewRecord(superusers)
-		rec.SetEmail(email)
+	existing, err := app.FindAuthRecordByEmail(superusers, email)
+	if err == nil && existing != nil {
+		return nil
 	}
-	rec.SetPassword(password)
-	return app.Save(rec)
+	record := core.NewRecord(superusers)
+	record.Email = email
+	if err := record.SetPassword(password); err != nil {
+		return err
+	}
+	return app.Save(record)
 }
-
-var errAppendOnly = errors.New("document_events is append-only: update and delete are refused")

archivist/internal/pbapp/setup_test.go

@@ -1,42 +1,112 @@
 package pbapp
 
 import (
+	"reflect"
 	"testing"
 
 	"ganigeorgiev.com/pocketbase/pocketbase"
+	"ganigeorgiev.com/pocketbase/pocketbase/core"
 	"github.com/napnap75/archivist/internal/config"
 )
 
-func TestEnsureDocumentsUsesTextForFreeFormEnums(t *testing.T) {
+func enumTestCfg(t *testing.T) *config.Config {
+	t.Helper()
+	tpl, err := config.CompileTemplate("default", "Divers/{documentType}.pdf")
+	if err != nil {
+		t.Fatalf("compile template: %v", err)
+	}
+	fields := []config.Field{
+		{Name: "template", Type: "enum"},
+		{Name: "documentType", Type: "enum"},
+	}
+	return &config.Config{
+		Fields: map[string]config.Field{
+			"template":     fields[0],
+			"documentType": fields[1],
+		},
+		FieldList: fields,
+		Templates: map[string]*config.CompiledTemplate{"default": tpl},
+	}
+}
+
+func enumTestApp(t *testing.T, cfg *config.Config) core.App {
+	t.Helper()
 	app := pocketbase.NewWithConfig(pocketbase.Config{DefaultDataDir: t.TempDir()})
 	if err := app.Bootstrap(); err != nil {
 		t.Fatalf("bootstrap PocketBase: %v", err)
 	}
-	defer app.ClearBootstrap()
+	if err := Setup(app, "admin@example.net", "password-123456", cfg); err != nil {
+		t.Fatalf("setup: %v", err)
+	}
+	return app
+}
 
-	fields := []config.Field{
-		{Name: "documentType", Type: "enum"},
-		{Name: "bankAccount", Type: "enum"},
-	}
-	cfg := &config.Config{
-		Fields: map[string]config.Field{
-			"documentType": fields[0],
-			"bankAccount":  fields[1],
-		},
-		FieldList: fields,
-	}
-	if err := ensureDocuments(app, cfg); err != nil {
-		t.Fatalf("ensure documents: %v", err)
-	}
+// US-03: declared enums become official select fields; the "template"
+// field starts with the configured template names, the others with the
+// placeholder.
+func TestEnsureDocumentsCreatesSelectEnums(t *testing.T) {
+	cfg := enumTestCfg(t)
+	app := enumTestApp(t, cfg)
+	defer app.ClearBootstrap()
 
 	coll, err := app.FindCollectionByNameOrId("documents")
 	if err != nil {
 		t.Fatalf("find documents collection: %v", err)
 	}
-	for _, name := range []string{"documentType", "bankAccount"} {
-		field := coll.Fields.GetByName(name)
-		if field == nil || field.Type() != "text" {
-			t.Errorf("field %q = %v, want text field", name, field)
+	for name, want := range map[string][]string{
+		"template":     {"default"},
+		"documentType": {DummyEnumValue},
+	} {
+		field, ok := coll.Fields.GetByName(name).(*core.SelectField)
+		if !ok || field == nil {
+			t.Fatalf("field %q is not a select field", name)
+		}
+		if !reflect.DeepEqual(field.Values, want) {
+			t.Errorf("field %q values = %v, want %v", name, field.Values, want)
 		}
 	}
 }
+
+// US-37: the startup cleanup keeps the values documents use, drops the
+// others, keeps the placeholder only while alone, and forces the template
+// list to the configuration.
+func TestCleanupEnumValues(t *testing.T) {
+	cfg := enumTestCfg(t)
+	app := enumTestApp(t, cfg)
+	defer app.ClearBootstrap()
+
+	if err := AddEnumValue(app, "documentType", "Facture"); err != nil {
+		t.Fatalf("add enum value: %v", err)
+	}
+	if err := AddEnumValue(app, "documentType", "obsolete"); err != nil {
+		t.Fatalf("add enum value: %v", err)
+	}
+	coll, err := app.FindCollectionByNameOrId("documents")
+	if err != nil {
+		t.Fatalf("find documents collection: %v", err)
+	}
+	rec := core.NewRecord(coll)
+	rec.Set("file_key", "pending/x.pdf")
+	rec.Set("status", "pending")
+	rec.Set("documentType", "Facture")
+	if err := app.Save(rec); err != nil {
+		t.Fatalf("save record: %v", err)
+	}
+
+	CleanupEnumValues(app, cfg)
+
+	values, err := enumFieldValues(app, "documentType")
+	if err != nil {
+		t.Fatalf("enum field values: %v", err)
+	}
+	if !reflect.DeepEqual(values, []string{"Facture"}) {
+		t.Errorf("documentType values = %v, want [Facture]", values)
+	}
+	tplValues, err := enumFieldValues(app, "template")
+	if err != nil {
+		t.Fatalf("enum field values: %v", err)
+	}
+	if !reflect.DeepEqual(tplValues, []string{"default"}) {
+		t.Errorf("template values = %v, want [default]", tplValues)
+	}
+}