Archivist : page Outils + outil « Recherche des fichiers perdus » (US-40, US-41) (#16)

* Add tools page with find-lost-files consistency tool (US-40, US-41)

Co-authored-by: napnap75 <napnap75@users.noreply.github.com>

* tools page: hide the job panel when there is nothing to show, guard against undefined rendering

Co-authored-by: napnap75 <napnap75@users.noreply.github.com>

* inbox: leave non-PDF objects in place, never imported (US-12)

Co-authored-by: napnap75 <napnap75@users.noreply.github.com>

---------

Co-authored-by: Vibe Nuage Agent <vibe@mistral.ai>
Co-authored-by: napnap75 <napnap75@users.noreply.github.com>
This commit is contained in:
2026-11-12 19:35:56 +01:00
committed by GitHub
co-authored by napnap75 Vibe Nuage Agent
parent 85784ef578
commit 1224ff916d
17 changed files with 1185 additions and 19 deletions
+55
View File
@@ -0,0 +1,55 @@
// Routes.go binds the tools-page application routes: the tool list, the
// run trigger, the live job state and the cancel action. Superuser-only,
// like every /api/archivist route.
package tools
import (
"ganigeorgiev.com/pocketbase/pocketbase/core"
)
// BindRoutes registers the tools routes on the PocketBase router.
func BindRoutes(app core.App, s3 S3Store) {
app.OnServe().BindFunc(func(e *core.ServeEvent) error {
e.Router.GET("/api/archivist/tools", func(re *core.RequestEvent) error {
if !re.HasSuperuserAuth() {
return re.ForbiddenError("superusers only", nil)
}
job := LatestJob()
var state map[string]any
if job != nil {
state = job.State()
}
return re.JSON(200, map[string]any{"tools": Descriptors(), "job": state})
})
e.Router.POST("/api/archivist/tools/{id}/run", func(re *core.RequestEvent) error {
if !re.HasSuperuserAuth() {
return re.ForbiddenError("superusers only", nil)
}
job, err := Start(re.App, s3, re.Request.PathValue("id"))
if err != nil {
return re.JSON(409, map[string]any{"message": err.Error()})
}
return re.JSON(200, job.State())
})
e.Router.GET("/api/archivist/tools/jobs/{id}", func(re *core.RequestEvent) error {
if !re.HasSuperuserAuth() {
return re.ForbiddenError("superusers only", nil)
}
job := JobByID(re.Request.PathValue("id"))
if job == nil {
return re.JSON(404, map[string]any{"message": "job not found"})
}
return re.JSON(200, job.State())
})
e.Router.POST("/api/archivist/tools/jobs/{id}/cancel", func(re *core.RequestEvent) error {
if !re.HasSuperuserAuth() {
return re.ForbiddenError("superusers only", nil)
}
if !Cancel(re.Request.PathValue("id")) {
return re.JSON(404, map[string]any{"message": "no running job with this id"})
}
return re.JSON(200, map[string]any{"status": "cancelling"})
})
return e.Next()
})
}
+393
View File
@@ -0,0 +1,393 @@
// Package tools implements the dashboard's tools page: long-running
// maintenance tasks started by hand, each executed as a background job with
// live progress and a final summary.
package tools
import (
"context"
"crypto/rand"
"database/sql"
"encoding/hex"
"errors"
"fmt"
"strings"
"sync"
"time"
"ganigeorgiev.com/pocketbase/dbx"
"ganigeorgiev.com/pocketbase/pocketbase/core"
"github.com/napnap75/archivist/internal/pbapp"
"github.com/napnap75/archivist/internal/storage"
)
// Descriptor is one tool as the tools page displays it: identity, title,
// description. The list is static; new tools are added here.
type Descriptor struct {
ID string `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
}
var registry = []Descriptor{
{
ID: "find-lost-files",
Name: "Recherche des fichiers perdus",
Description: "Parcourt tout le bucket (hors inbox/), recalcule l'empreinte sha256 de chaque objet et la compare à la base. " +
"Un objet à sa place connue ne bouge pas ; un objet déplacé hors de l'application voit son file_key corrigé avec un événement relocated dans l'historique ; " +
"un objet inconnu de la base est déplacé vers inbox/ pour être traité automatiquement ; " +
"les enregistrements dont le fichier a disparu du bucket sont signalés dans le résumé, sans action automatique.",
},
}
// Descriptors returns the tool list served to the dashboard.
func Descriptors() []Descriptor {
return registry
}
func descriptor(id string) (Descriptor, bool) {
for _, d := range registry {
if d.ID == id {
return d, true
}
}
return Descriptor{}, false
}
// S3Store is the surface of the storage client the tools use; a seam kept
// for unit tests.
type S3Store interface {
ListAll(ctx context.Context) ([]storage.InboxObject, error)
Hash(ctx context.Context, key string) (string, error)
Exists(ctx context.Context, key string) bool
Copy(ctx context.Context, srcKey, dstKey string) error
Remove(ctx context.Context, key string) error
ResolveConflict(ctx context.Context, desired string) (string, error)
}
// Job is one tool run: identity, live progress, final summary.
type Job struct {
ID string
Tool string
Status string // running | done | failed | cancelled
StartedAt time.Time
FinishedAt time.Time
Total int
Processed int
Current string
Summary map[string]int
Missing []string
Errors []string
mu sync.Mutex
cancel context.CancelFunc
}
const (
StatusRunning = "running"
StatusDone = "done"
StatusFailed = "failed"
StatusCancelled = "cancelled"
)
// State snapshots the job for the API responses.
func (j *Job) State() map[string]any {
j.mu.Lock()
defer j.mu.Unlock()
state := map[string]any{
"id": j.ID,
"tool": j.Tool,
"status": j.Status,
"started_at": j.StartedAt.Format(time.RFC3339),
"total": j.Total,
"processed": j.Processed,
"current": j.Current,
"summary": j.Summary,
"missing": j.Missing,
"errors": j.Errors,
}
if !j.FinishedAt.IsZero() {
state["finished_at"] = j.FinishedAt.Format(time.RFC3339)
}
return state
}
func (j *Job) set(fn func()) {
j.mu.Lock()
fn()
j.mu.Unlock()
}
func (j *Job) addError(err error, key string) {
j.set(func() {
j.Errors = append(j.Errors, key+": "+err.Error())
if n, ok := j.Summary["errors"]; ok {
j.Summary["errors"] = n + 1
} else {
j.Summary["errors"] = 1
}
})
}
var (
regMu sync.Mutex
jobs = map[string]*Job{}
running *Job
)
// Start launches a tool as a background job. One job at a time: a second
// start while one is running is refused.
func Start(app core.App, s3 S3Store, toolID string) (*Job, error) {
d, ok := descriptor(toolID)
if !ok {
return nil, fmt.Errorf("unknown tool: %s", toolID)
}
regMu.Lock()
defer regMu.Unlock()
if running != nil {
return nil, fmt.Errorf("a job is already running: %s", running.Tool)
}
buf := make([]byte, 8)
if _, err := rand.Read(buf); err != nil {
return nil, err
}
ctx, cancel := context.WithCancel(context.Background())
job := &Job{
ID: hex.EncodeToString(buf),
Tool: d.ID,
Status: StatusRunning,
StartedAt: time.Now().UTC(),
Summary: map[string]int{},
cancel: cancel,
}
if d.ID == "find-lost-files" {
job.Summary = map[string]int{"scanned": 0, "ok": 0, "corrected": 0, "moved": 0, "duplicates": 0, "errors": 0}
}
jobs[job.ID] = job
running = job
go func() {
err := runTool(ctx, app, s3, d.ID, job)
regMu.Lock()
if running == job {
running = nil
}
regMu.Unlock()
job.set(func() {
job.cancel = nil
job.FinishedAt = time.Now().UTC()
if ctx.Err() != nil {
job.Status = StatusCancelled
} else if err != nil {
job.Status = StatusFailed
job.Errors = append(job.Errors, err.Error())
} else {
job.Status = StatusDone
}
job.Current = ""
})
}()
return job, nil
}
// Cancel asks the running job to stop at the next object boundary.
func Cancel(jobID string) bool {
regMu.Lock()
job := jobs[jobID]
regMu.Unlock()
if job == nil {
return false
}
job.mu.Lock()
cancel := job.cancel
job.mu.Unlock()
if cancel == nil {
return false
}
cancel()
return true
}
// JobByID returns a job by id, nil when unknown.
func JobByID(id string) *Job {
regMu.Lock()
defer regMu.Unlock()
return jobs[id]
}
// LatestJob returns the running job if any, otherwise the most recent one
// (the tools page restores its panel on reload).
func LatestJob() *Job {
regMu.Lock()
defer regMu.Unlock()
if running != nil {
return running
}
var latest *Job
for _, j := range jobs {
if latest == nil || j.StartedAt.After(latest.StartedAt) {
latest = j
}
}
return latest
}
func runTool(ctx context.Context, app core.App, s3 S3Store, toolID string, job *Job) error {
switch toolID {
case "find-lost-files":
return runFindLostFiles(ctx, app, s3, job)
}
return fmt.Errorf("unknown tool: %s", toolID)
}
// runFindLostFiles reconciles the bucket with the database (US-41): every
// object outside inbox/ is hashed and matched against documents by sha256.
// In place: nothing. Moved in the bucket outside the application: the
// record's file_key is corrected and a relocated event is appended. Unknown
// of the database: moved to inbox/ for automatic processing. A record whose
// file_key exists nowhere is only reported.
func runFindLostFiles(ctx context.Context, app core.App, s3 S3Store, job *Job) error {
objects, err := s3.ListAll(ctx)
if err != nil {
return fmt.Errorf("list bucket: %w", err)
}
keySet := make(map[string]bool, len(objects))
for _, o := range objects {
keySet[o.Key] = true
}
job.set(func() { job.Total = len(objects) })
for _, o := range objects {
if ctx.Err() != nil {
return nil
}
job.set(func() { job.Current = o.Key })
if err := reconcileObject(ctx, app, s3, job, o.Key, keySet); err != nil {
app.Logger().Error("find-lost-files: object failed", "key", o.Key, "error", err)
job.addError(err, o.Key)
}
job.set(func() { job.Processed++ })
}
// Records whose file_key exists nowhere: reported, never acted upon.
// Records still under inbox/ are interrupted filings (US-11), the
// poller's business, not orphans.
records, err := app.FindAllRecords("documents")
if err != nil {
return fmt.Errorf("list documents: %w", err)
}
missing := []string{}
for _, r := range records {
key := r.GetString("file_key")
if key == "" || keySet[key] || strings.HasPrefix(key, "inbox/") {
continue
}
missing = append(missing, key)
app.Logger().Warn("find-lost-files: record file missing from bucket", "record", r.Id, "file_key", key)
}
job.set(func() {
job.Missing = missing
job.Summary["missing"] = len(missing)
})
job.set(func() {
app.Logger().Info("find-lost-files: done", "scanned", job.Summary["scanned"], "ok", job.Summary["ok"],
"corrected", job.Summary["corrected"], "moved", job.Summary["moved"],
"duplicates", job.Summary["duplicates"], "errors", job.Summary["errors"], "missing", len(missing))
})
return nil
}
// findRecord wraps FindFirstRecordByFilter: a missing record is a nil
// result, not an error.
func findRecord(app core.App, filter string, params dbx.Params) (*core.Record, error) {
record, err := app.FindFirstRecordByFilter("documents", filter, params)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return nil, err
}
return record, nil
}
func reconcileObject(ctx context.Context, app core.App, s3 S3Store, job *Job, key string, keySet map[string]bool) error {
hash, err := s3.Hash(ctx, key)
if err != nil {
return fmt.Errorf("hash: %w", err)
}
job.set(func() { job.Summary["scanned"]++ })
record, err := findRecord(app, "sha256 = {:h}", dbx.Params{"h": hash})
if err != nil {
return fmt.Errorf("lookup by sha256: %w", err)
}
if record == nil {
// Unknown of the database: back to inbox/ for automatic processing.
finalKey, err := s3.ResolveConflict(ctx, "inbox/"+baseName(key))
if err != nil {
return fmt.Errorf("conflict resolution: %w", err)
}
if err := moveObject(ctx, s3, key, finalKey); err != nil {
return err
}
job.set(func() { job.Summary["moved"]++ })
app.Logger().Info("find-lost-files: unknown object moved to inbox", "from", key, "to", finalKey)
return nil
}
oldKey := record.GetString("file_key")
if oldKey == key {
job.set(func() { job.Summary["ok"]++ })
return nil
}
if keySet[oldKey] {
// The record points at an object that still exists: this scanned
// object is a stray copy, left in place and reported.
job.set(func() { job.Summary["duplicates"]++ })
app.Logger().Warn("find-lost-files: stray copy left in place", "key", key, "record", record.Id, "file_key", oldKey)
return nil
}
// The file was moved in the bucket outside the application: the record
// follows it, a relocated event keeps the trace. No other record may
// already reference the new key.
other, err := findRecord(app, "file_key = {:k} && id != {:id}", dbx.Params{"k": key, "id": record.Id})
if err != nil {
return fmt.Errorf("lookup by file_key: %w", err)
}
if other != nil {
return fmt.Errorf("another record (%s) already references %s", other.Id, key)
}
before := pbapp.Snapshot{"file_key": oldKey}
after := pbapp.Snapshot{"file_key": key}
txErr := app.RunInTransaction(func(tx core.App) error {
record.Set("file_key", key)
if err := tx.Save(record); err != nil {
return err
}
return pbapp.AppendEvent(tx, record.Id, "relocated", before, after, nil)
})
if txErr != nil {
return fmt.Errorf("relocation save: %w", txErr)
}
job.set(func() { job.Summary["corrected"]++ })
app.Logger().Info("find-lost-files: file_key corrected", "record", record.Id, "from", oldKey, "to", key)
return nil
}
// moveObject copies src to dst, verifies the copy, then deletes the source.
func moveObject(ctx context.Context, s3 S3Store, src, dst string) error {
if err := s3.Copy(ctx, src, dst); err != nil {
return fmt.Errorf("copy: %w", err)
}
if !s3.Exists(ctx, dst) {
return fmt.Errorf("target %s not verified after copy", dst)
}
if err := s3.Remove(ctx, src); err != nil {
return fmt.Errorf("remove source: %w", err)
}
return nil
}
func baseName(key string) string {
if i := strings.LastIndex(key, "/"); i >= 0 {
return key[i+1:]
}
return key
}
+274
View File
@@ -0,0 +1,274 @@
package tools
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"sort"
"strings"
"testing"
"time"
"ganigeorgiev.com/pocketbase/pocketbase"
"ganigeorgiev.com/pocketbase/pocketbase/core"
"github.com/napnap75/archivist/internal/config"
"github.com/napnap75/archivist/internal/pbapp"
"github.com/napnap75/archivist/internal/storage"
)
// fakeS3 is an in-memory bucket for the tool tests.
type fakeS3 struct {
objects map[string][]byte
}
func (f *fakeS3) ListAll(ctx context.Context) ([]storage.InboxObject, error) {
var keys []string
for k := range f.objects {
if strings.HasPrefix(k, "inbox/") {
continue
}
keys = append(keys, k)
}
sort.Strings(keys)
out := make([]storage.InboxObject, 0, len(keys))
for _, k := range keys {
out = append(out, storage.InboxObject{Key: k, Size: int64(len(f.objects[k]))})
}
return out, nil
}
func (f *fakeS3) Hash(ctx context.Context, key string) (string, error) {
data, ok := f.objects[key]
if !ok {
return "", fmt.Errorf("no such key %q", key)
}
sum := sha256.Sum256(data)
return hex.EncodeToString(sum[:]), nil
}
func (f *fakeS3) Exists(ctx context.Context, key string) bool {
_, ok := f.objects[key]
return ok
}
func (f *fakeS3) Copy(ctx context.Context, srcKey, dstKey string) error {
data, ok := f.objects[srcKey]
if !ok {
return fmt.Errorf("no such key %q", srcKey)
}
f.objects[dstKey] = data
return nil
}
func (f *fakeS3) Remove(ctx context.Context, key string) error {
delete(f.objects, key)
return nil
}
func (f *fakeS3) ResolveConflict(ctx context.Context, desired string) (string, error) {
if !f.Exists(ctx, desired) {
return desired, nil
}
for n := 1; n < 1000; n++ {
ext := ""
if i := strings.LastIndex(desired, "."); i >= 0 {
ext = desired[i:]
}
candidate := fmt.Sprintf("%s (%d)%s", strings.TrimSuffix(desired, ext), n, ext)
if !f.Exists(ctx, candidate) {
return candidate, nil
}
}
return "", fmt.Errorf("cannot resolve conflict on %q", desired)
}
func newTestApp(t *testing.T) core.App {
t.Helper()
app := pocketbase.NewWithConfig(pocketbase.Config{DefaultDataDir: t.TempDir()})
if err := app.Bootstrap(); err != nil {
t.Fatalf("bootstrap PocketBase: %v", err)
}
t.Cleanup(func() { app.ClearBootstrap() })
cfg := &config.Config{
Fields: map[string]config.Field{
"documentType": {Name: "documentType", Type: "text"},
},
FieldList: []config.Field{{Name: "documentType", Type: "text"}},
}
if err := pbapp.Setup(app, "admin@example.net", "password-123456", cfg); err != nil {
t.Fatalf("setup: %v", err)
}
return app
}
func addDocument(t *testing.T, app core.App, sha, fileKey string) *core.Record {
t.Helper()
coll, err := app.FindCollectionByNameOrId("documents")
if err != nil {
t.Fatalf("find documents: %v", err)
}
rec := core.NewRecord(coll)
rec.Set("sha256", sha)
rec.Set("file_key", fileKey)
rec.Set("status", "processed")
if err := app.Save(rec); err != nil {
t.Fatalf("save document: %v", err)
}
return rec
}
func runScan(t *testing.T, app core.App, s3 S3Store) *Job {
t.Helper()
job := &Job{
Tool: "find-lost-files",
Status: StatusRunning,
StartedAt: time.Now().UTC(),
Summary: map[string]int{"scanned": 0, "ok": 0, "corrected": 0, "moved": 0, "duplicates": 0, "errors": 0},
}
if err := runFindLostFiles(context.Background(), app, s3, job); err != nil {
t.Fatalf("runFindLostFiles: %v", err)
}
return job
}
func TestFindLostFilesInPlace(t *testing.T) {
app := newTestApp(t)
s3 := &fakeS3{objects: map[string][]byte{"files/a.pdf": []byte("content-a")}}
addDocument(t, app, mustHash("content-a"), "files/a.pdf")
job := runScan(t, app, s3)
if job.Summary["ok"] != 1 || job.Summary["corrected"] != 0 || job.Summary["moved"] != 0 {
t.Fatalf("summary = %v, want ok=1", job.Summary)
}
if len(job.Missing) != 0 {
t.Fatalf("missing = %v, want empty", job.Missing)
}
if _, ok := s3.objects["files/a.pdf"]; !ok {
t.Fatalf("object moved: %v", s3.objects)
}
}
func TestFindLostFilesRelocated(t *testing.T) {
app := newTestApp(t)
s3 := &fakeS3{objects: map[string][]byte{"files/new/a.pdf": []byte("content-a")}}
rec := addDocument(t, app, mustHash("content-a"), "files/old/a.pdf")
job := runScan(t, app, s3)
if job.Summary["corrected"] != 1 {
t.Fatalf("summary = %v, want corrected=1", job.Summary)
}
fresh, err := app.FindRecordById("documents", rec.Id)
if err != nil {
t.Fatal(err)
}
if got := fresh.GetString("file_key"); got != "files/new/a.pdf" {
t.Fatalf("file_key = %q, want files/new/a.pdf", got)
}
events, err := app.FindAllRecords("document_events")
if err != nil {
t.Fatal(err)
}
if len(events) != 1 {
t.Fatalf("events = %d, want 1", len(events))
}
if events[0].GetString("event_type") != "relocated" {
t.Fatalf("event_type = %q, want relocated", events[0].GetString("event_type"))
}
if _, ok := s3.objects["files/new/a.pdf"]; !ok {
t.Fatalf("object should not move: %v", s3.objects)
}
}
func TestFindLostFilesUnknownGoesToInbox(t *testing.T) {
app := newTestApp(t)
s3 := &fakeS3{objects: map[string][]byte{"files/unknown.pdf": []byte("content-x")}}
job := runScan(t, app, s3)
if job.Summary["moved"] != 1 {
t.Fatalf("summary = %v, want moved=1", job.Summary)
}
if _, ok := s3.objects["inbox/unknown.pdf"]; !ok {
t.Fatalf("object not in inbox: %v", s3.objects)
}
}
func TestFindLostFilesStrayDuplicateLeftInPlace(t *testing.T) {
app := newTestApp(t)
s3 := &fakeS3{objects: map[string][]byte{
"files/orig.pdf": []byte("content-a"),
"files/stray.pdf": []byte("content-a"),
}}
addDocument(t, app, mustHash("content-a"), "files/orig.pdf")
job := runScan(t, app, s3)
if job.Summary["duplicates"] != 1 || job.Summary["ok"] != 1 {
t.Fatalf("summary = %v, want duplicates=1 ok=1", job.Summary)
}
if _, ok := s3.objects["files/stray.pdf"]; !ok {
t.Fatalf("stray copy should stay: %v", s3.objects)
}
}
func TestFindLostFilesMissingReported(t *testing.T) {
app := newTestApp(t)
s3 := &fakeS3{objects: map[string][]byte{}}
addDocument(t, app, mustHash("content-a"), "files/gone.pdf")
job := runScan(t, app, s3)
if len(job.Missing) != 1 || job.Missing[0] != "files/gone.pdf" {
t.Fatalf("missing = %v, want [files/gone.pdf]", job.Missing)
}
if job.Summary["missing"] != 1 {
t.Fatalf("summary = %v, want missing=1", job.Summary)
}
}
func TestFindLostFilesInboxExcluded(t *testing.T) {
app := newTestApp(t)
s3 := &fakeS3{objects: map[string][]byte{
"inbox/upload.pdf": []byte("content-inbox"),
"files/a.pdf": []byte("content-a"),
}}
addDocument(t, app, mustHash("content-a"), "files/a.pdf")
addDocument(t, app, mustHash("content-inbox"), "inbox/upload.pdf")
job := runScan(t, app, s3)
if job.Summary["scanned"] != 1 {
t.Fatalf("summary = %v, want scanned=1 (inbox excluded)", job.Summary)
}
if len(job.Missing) != 0 {
t.Fatalf("missing = %v, want empty (inbox records are the poller's business)", job.Missing)
}
if _, ok := s3.objects["inbox/upload.pdf"]; !ok {
t.Fatalf("inbox object should stay: %v", s3.objects)
}
}
func TestEventTypesIncludeRelocated(t *testing.T) {
app := newTestApp(t)
coll, err := app.FindCollectionByNameOrId("document_events")
if err != nil {
t.Fatal(err)
}
field, ok := coll.Fields.GetByName("event_type").(*core.SelectField)
if !ok {
t.Fatalf("event_type is not a select field")
}
for _, v := range []string{"imported", "corrected", "validated", "relocated"} {
found := false
for _, value := range field.Values {
if value == v {
found = true
}
}
if !found {
t.Errorf("event_type values %v miss %q", field.Values, v)
}
}
}
func mustHash(content string) string {
sum := sha256.Sum256([]byte(content))
return hex.EncodeToString(sum[:])
}