mirror of
https://github.com/napnap75/multiarch-docker-images.git
synced 2026-11-16 19:01:48 +01:00
Archivist : page Outils + outil « Recherche des fichiers perdus » (US-40, US-41) (#16)
* Add tools page with find-lost-files consistency tool (US-40, US-41) Co-authored-by: napnap75 <napnap75@users.noreply.github.com> * tools page: hide the job panel when there is nothing to show, guard against undefined rendering Co-authored-by: napnap75 <napnap75@users.noreply.github.com> * inbox: leave non-PDF objects in place, never imported (US-12) Co-authored-by: napnap75 <napnap75@users.noreply.github.com> --------- Co-authored-by: Vibe Nuage Agent <vibe@mistral.ai> Co-authored-by: napnap75 <napnap75@users.noreply.github.com>
This commit is contained in:
co-authored by
napnap75
Vibe Nuage Agent
parent
85784ef578
commit
1224ff916d
@@ -0,0 +1,55 @@
|
||||
// Routes.go binds the tools-page application routes: the tool list, the
|
||||
// run trigger, the live job state and the cancel action. Superuser-only,
|
||||
// like every /api/archivist route.
|
||||
package tools
|
||||
|
||||
import (
|
||||
"ganigeorgiev.com/pocketbase/pocketbase/core"
|
||||
)
|
||||
|
||||
// BindRoutes registers the tools routes on the PocketBase router.
|
||||
func BindRoutes(app core.App, s3 S3Store) {
|
||||
app.OnServe().BindFunc(func(e *core.ServeEvent) error {
|
||||
e.Router.GET("/api/archivist/tools", func(re *core.RequestEvent) error {
|
||||
if !re.HasSuperuserAuth() {
|
||||
return re.ForbiddenError("superusers only", nil)
|
||||
}
|
||||
job := LatestJob()
|
||||
var state map[string]any
|
||||
if job != nil {
|
||||
state = job.State()
|
||||
}
|
||||
return re.JSON(200, map[string]any{"tools": Descriptors(), "job": state})
|
||||
})
|
||||
e.Router.POST("/api/archivist/tools/{id}/run", func(re *core.RequestEvent) error {
|
||||
if !re.HasSuperuserAuth() {
|
||||
return re.ForbiddenError("superusers only", nil)
|
||||
}
|
||||
job, err := Start(re.App, s3, re.Request.PathValue("id"))
|
||||
if err != nil {
|
||||
return re.JSON(409, map[string]any{"message": err.Error()})
|
||||
}
|
||||
return re.JSON(200, job.State())
|
||||
})
|
||||
e.Router.GET("/api/archivist/tools/jobs/{id}", func(re *core.RequestEvent) error {
|
||||
if !re.HasSuperuserAuth() {
|
||||
return re.ForbiddenError("superusers only", nil)
|
||||
}
|
||||
job := JobByID(re.Request.PathValue("id"))
|
||||
if job == nil {
|
||||
return re.JSON(404, map[string]any{"message": "job not found"})
|
||||
}
|
||||
return re.JSON(200, job.State())
|
||||
})
|
||||
e.Router.POST("/api/archivist/tools/jobs/{id}/cancel", func(re *core.RequestEvent) error {
|
||||
if !re.HasSuperuserAuth() {
|
||||
return re.ForbiddenError("superusers only", nil)
|
||||
}
|
||||
if !Cancel(re.Request.PathValue("id")) {
|
||||
return re.JSON(404, map[string]any{"message": "no running job with this id"})
|
||||
}
|
||||
return re.JSON(200, map[string]any{"status": "cancelling"})
|
||||
})
|
||||
return e.Next()
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,393 @@
|
||||
// Package tools implements the dashboard's tools page: long-running
|
||||
// maintenance tasks started by hand, each executed as a background job with
|
||||
// live progress and a final summary.
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"ganigeorgiev.com/pocketbase/dbx"
|
||||
"ganigeorgiev.com/pocketbase/pocketbase/core"
|
||||
"github.com/napnap75/archivist/internal/pbapp"
|
||||
"github.com/napnap75/archivist/internal/storage"
|
||||
)
|
||||
|
||||
// Descriptor is one tool as the tools page displays it: identity, title,
|
||||
// description. The list is static; new tools are added here.
|
||||
type Descriptor struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
var registry = []Descriptor{
|
||||
{
|
||||
ID: "find-lost-files",
|
||||
Name: "Recherche des fichiers perdus",
|
||||
Description: "Parcourt tout le bucket (hors inbox/), recalcule l'empreinte sha256 de chaque objet et la compare à la base. " +
|
||||
"Un objet à sa place connue ne bouge pas ; un objet déplacé hors de l'application voit son file_key corrigé avec un événement relocated dans l'historique ; " +
|
||||
"un objet inconnu de la base est déplacé vers inbox/ pour être traité automatiquement ; " +
|
||||
"les enregistrements dont le fichier a disparu du bucket sont signalés dans le résumé, sans action automatique.",
|
||||
},
|
||||
}
|
||||
|
||||
// Descriptors returns the tool list served to the dashboard.
|
||||
func Descriptors() []Descriptor {
|
||||
return registry
|
||||
}
|
||||
|
||||
func descriptor(id string) (Descriptor, bool) {
|
||||
for _, d := range registry {
|
||||
if d.ID == id {
|
||||
return d, true
|
||||
}
|
||||
}
|
||||
return Descriptor{}, false
|
||||
}
|
||||
|
||||
// S3Store is the surface of the storage client the tools use; a seam kept
|
||||
// for unit tests.
|
||||
type S3Store interface {
|
||||
ListAll(ctx context.Context) ([]storage.InboxObject, error)
|
||||
Hash(ctx context.Context, key string) (string, error)
|
||||
Exists(ctx context.Context, key string) bool
|
||||
Copy(ctx context.Context, srcKey, dstKey string) error
|
||||
Remove(ctx context.Context, key string) error
|
||||
ResolveConflict(ctx context.Context, desired string) (string, error)
|
||||
}
|
||||
|
||||
// Job is one tool run: identity, live progress, final summary.
|
||||
type Job struct {
|
||||
ID string
|
||||
Tool string
|
||||
Status string // running | done | failed | cancelled
|
||||
StartedAt time.Time
|
||||
FinishedAt time.Time
|
||||
Total int
|
||||
Processed int
|
||||
Current string
|
||||
Summary map[string]int
|
||||
Missing []string
|
||||
Errors []string
|
||||
|
||||
mu sync.Mutex
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
const (
|
||||
StatusRunning = "running"
|
||||
StatusDone = "done"
|
||||
StatusFailed = "failed"
|
||||
StatusCancelled = "cancelled"
|
||||
)
|
||||
|
||||
// State snapshots the job for the API responses.
|
||||
func (j *Job) State() map[string]any {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
state := map[string]any{
|
||||
"id": j.ID,
|
||||
"tool": j.Tool,
|
||||
"status": j.Status,
|
||||
"started_at": j.StartedAt.Format(time.RFC3339),
|
||||
"total": j.Total,
|
||||
"processed": j.Processed,
|
||||
"current": j.Current,
|
||||
"summary": j.Summary,
|
||||
"missing": j.Missing,
|
||||
"errors": j.Errors,
|
||||
}
|
||||
if !j.FinishedAt.IsZero() {
|
||||
state["finished_at"] = j.FinishedAt.Format(time.RFC3339)
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
func (j *Job) set(fn func()) {
|
||||
j.mu.Lock()
|
||||
fn()
|
||||
j.mu.Unlock()
|
||||
}
|
||||
|
||||
func (j *Job) addError(err error, key string) {
|
||||
j.set(func() {
|
||||
j.Errors = append(j.Errors, key+": "+err.Error())
|
||||
if n, ok := j.Summary["errors"]; ok {
|
||||
j.Summary["errors"] = n + 1
|
||||
} else {
|
||||
j.Summary["errors"] = 1
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
var (
|
||||
regMu sync.Mutex
|
||||
jobs = map[string]*Job{}
|
||||
running *Job
|
||||
)
|
||||
|
||||
// Start launches a tool as a background job. One job at a time: a second
|
||||
// start while one is running is refused.
|
||||
func Start(app core.App, s3 S3Store, toolID string) (*Job, error) {
|
||||
d, ok := descriptor(toolID)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unknown tool: %s", toolID)
|
||||
}
|
||||
regMu.Lock()
|
||||
defer regMu.Unlock()
|
||||
if running != nil {
|
||||
return nil, fmt.Errorf("a job is already running: %s", running.Tool)
|
||||
}
|
||||
buf := make([]byte, 8)
|
||||
if _, err := rand.Read(buf); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
job := &Job{
|
||||
ID: hex.EncodeToString(buf),
|
||||
Tool: d.ID,
|
||||
Status: StatusRunning,
|
||||
StartedAt: time.Now().UTC(),
|
||||
Summary: map[string]int{},
|
||||
cancel: cancel,
|
||||
}
|
||||
if d.ID == "find-lost-files" {
|
||||
job.Summary = map[string]int{"scanned": 0, "ok": 0, "corrected": 0, "moved": 0, "duplicates": 0, "errors": 0}
|
||||
}
|
||||
jobs[job.ID] = job
|
||||
running = job
|
||||
go func() {
|
||||
err := runTool(ctx, app, s3, d.ID, job)
|
||||
regMu.Lock()
|
||||
if running == job {
|
||||
running = nil
|
||||
}
|
||||
regMu.Unlock()
|
||||
job.set(func() {
|
||||
job.cancel = nil
|
||||
job.FinishedAt = time.Now().UTC()
|
||||
if ctx.Err() != nil {
|
||||
job.Status = StatusCancelled
|
||||
} else if err != nil {
|
||||
job.Status = StatusFailed
|
||||
job.Errors = append(job.Errors, err.Error())
|
||||
} else {
|
||||
job.Status = StatusDone
|
||||
}
|
||||
job.Current = ""
|
||||
})
|
||||
}()
|
||||
return job, nil
|
||||
}
|
||||
|
||||
// Cancel asks the running job to stop at the next object boundary.
|
||||
func Cancel(jobID string) bool {
|
||||
regMu.Lock()
|
||||
job := jobs[jobID]
|
||||
regMu.Unlock()
|
||||
if job == nil {
|
||||
return false
|
||||
}
|
||||
job.mu.Lock()
|
||||
cancel := job.cancel
|
||||
job.mu.Unlock()
|
||||
if cancel == nil {
|
||||
return false
|
||||
}
|
||||
cancel()
|
||||
return true
|
||||
}
|
||||
|
||||
// JobByID returns a job by id, nil when unknown.
|
||||
func JobByID(id string) *Job {
|
||||
regMu.Lock()
|
||||
defer regMu.Unlock()
|
||||
return jobs[id]
|
||||
}
|
||||
|
||||
// LatestJob returns the running job if any, otherwise the most recent one
|
||||
// (the tools page restores its panel on reload).
|
||||
func LatestJob() *Job {
|
||||
regMu.Lock()
|
||||
defer regMu.Unlock()
|
||||
if running != nil {
|
||||
return running
|
||||
}
|
||||
var latest *Job
|
||||
for _, j := range jobs {
|
||||
if latest == nil || j.StartedAt.After(latest.StartedAt) {
|
||||
latest = j
|
||||
}
|
||||
}
|
||||
return latest
|
||||
}
|
||||
|
||||
func runTool(ctx context.Context, app core.App, s3 S3Store, toolID string, job *Job) error {
|
||||
switch toolID {
|
||||
case "find-lost-files":
|
||||
return runFindLostFiles(ctx, app, s3, job)
|
||||
}
|
||||
return fmt.Errorf("unknown tool: %s", toolID)
|
||||
}
|
||||
|
||||
// runFindLostFiles reconciles the bucket with the database (US-41): every
|
||||
// object outside inbox/ is hashed and matched against documents by sha256.
|
||||
// In place: nothing. Moved in the bucket outside the application: the
|
||||
// record's file_key is corrected and a relocated event is appended. Unknown
|
||||
// of the database: moved to inbox/ for automatic processing. A record whose
|
||||
// file_key exists nowhere is only reported.
|
||||
func runFindLostFiles(ctx context.Context, app core.App, s3 S3Store, job *Job) error {
|
||||
objects, err := s3.ListAll(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list bucket: %w", err)
|
||||
}
|
||||
keySet := make(map[string]bool, len(objects))
|
||||
for _, o := range objects {
|
||||
keySet[o.Key] = true
|
||||
}
|
||||
job.set(func() { job.Total = len(objects) })
|
||||
|
||||
for _, o := range objects {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
job.set(func() { job.Current = o.Key })
|
||||
if err := reconcileObject(ctx, app, s3, job, o.Key, keySet); err != nil {
|
||||
app.Logger().Error("find-lost-files: object failed", "key", o.Key, "error", err)
|
||||
job.addError(err, o.Key)
|
||||
}
|
||||
job.set(func() { job.Processed++ })
|
||||
}
|
||||
|
||||
// Records whose file_key exists nowhere: reported, never acted upon.
|
||||
// Records still under inbox/ are interrupted filings (US-11), the
|
||||
// poller's business, not orphans.
|
||||
records, err := app.FindAllRecords("documents")
|
||||
if err != nil {
|
||||
return fmt.Errorf("list documents: %w", err)
|
||||
}
|
||||
missing := []string{}
|
||||
for _, r := range records {
|
||||
key := r.GetString("file_key")
|
||||
if key == "" || keySet[key] || strings.HasPrefix(key, "inbox/") {
|
||||
continue
|
||||
}
|
||||
missing = append(missing, key)
|
||||
app.Logger().Warn("find-lost-files: record file missing from bucket", "record", r.Id, "file_key", key)
|
||||
}
|
||||
job.set(func() {
|
||||
job.Missing = missing
|
||||
job.Summary["missing"] = len(missing)
|
||||
})
|
||||
job.set(func() {
|
||||
app.Logger().Info("find-lost-files: done", "scanned", job.Summary["scanned"], "ok", job.Summary["ok"],
|
||||
"corrected", job.Summary["corrected"], "moved", job.Summary["moved"],
|
||||
"duplicates", job.Summary["duplicates"], "errors", job.Summary["errors"], "missing", len(missing))
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// findRecord wraps FindFirstRecordByFilter: a missing record is a nil
|
||||
// result, not an error.
|
||||
func findRecord(app core.App, filter string, params dbx.Params) (*core.Record, error) {
|
||||
record, err := app.FindFirstRecordByFilter("documents", filter, params)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, err
|
||||
}
|
||||
return record, nil
|
||||
}
|
||||
|
||||
func reconcileObject(ctx context.Context, app core.App, s3 S3Store, job *Job, key string, keySet map[string]bool) error {
|
||||
hash, err := s3.Hash(ctx, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash: %w", err)
|
||||
}
|
||||
job.set(func() { job.Summary["scanned"]++ })
|
||||
|
||||
record, err := findRecord(app, "sha256 = {:h}", dbx.Params{"h": hash})
|
||||
if err != nil {
|
||||
return fmt.Errorf("lookup by sha256: %w", err)
|
||||
}
|
||||
|
||||
if record == nil {
|
||||
// Unknown of the database: back to inbox/ for automatic processing.
|
||||
finalKey, err := s3.ResolveConflict(ctx, "inbox/"+baseName(key))
|
||||
if err != nil {
|
||||
return fmt.Errorf("conflict resolution: %w", err)
|
||||
}
|
||||
if err := moveObject(ctx, s3, key, finalKey); err != nil {
|
||||
return err
|
||||
}
|
||||
job.set(func() { job.Summary["moved"]++ })
|
||||
app.Logger().Info("find-lost-files: unknown object moved to inbox", "from", key, "to", finalKey)
|
||||
return nil
|
||||
}
|
||||
|
||||
oldKey := record.GetString("file_key")
|
||||
if oldKey == key {
|
||||
job.set(func() { job.Summary["ok"]++ })
|
||||
return nil
|
||||
}
|
||||
if keySet[oldKey] {
|
||||
// The record points at an object that still exists: this scanned
|
||||
// object is a stray copy, left in place and reported.
|
||||
job.set(func() { job.Summary["duplicates"]++ })
|
||||
app.Logger().Warn("find-lost-files: stray copy left in place", "key", key, "record", record.Id, "file_key", oldKey)
|
||||
return nil
|
||||
}
|
||||
|
||||
// The file was moved in the bucket outside the application: the record
|
||||
// follows it, a relocated event keeps the trace. No other record may
|
||||
// already reference the new key.
|
||||
other, err := findRecord(app, "file_key = {:k} && id != {:id}", dbx.Params{"k": key, "id": record.Id})
|
||||
if err != nil {
|
||||
return fmt.Errorf("lookup by file_key: %w", err)
|
||||
}
|
||||
if other != nil {
|
||||
return fmt.Errorf("another record (%s) already references %s", other.Id, key)
|
||||
}
|
||||
before := pbapp.Snapshot{"file_key": oldKey}
|
||||
after := pbapp.Snapshot{"file_key": key}
|
||||
txErr := app.RunInTransaction(func(tx core.App) error {
|
||||
record.Set("file_key", key)
|
||||
if err := tx.Save(record); err != nil {
|
||||
return err
|
||||
}
|
||||
return pbapp.AppendEvent(tx, record.Id, "relocated", before, after, nil)
|
||||
})
|
||||
if txErr != nil {
|
||||
return fmt.Errorf("relocation save: %w", txErr)
|
||||
}
|
||||
job.set(func() { job.Summary["corrected"]++ })
|
||||
app.Logger().Info("find-lost-files: file_key corrected", "record", record.Id, "from", oldKey, "to", key)
|
||||
return nil
|
||||
}
|
||||
|
||||
// moveObject copies src to dst, verifies the copy, then deletes the source.
|
||||
func moveObject(ctx context.Context, s3 S3Store, src, dst string) error {
|
||||
if err := s3.Copy(ctx, src, dst); err != nil {
|
||||
return fmt.Errorf("copy: %w", err)
|
||||
}
|
||||
if !s3.Exists(ctx, dst) {
|
||||
return fmt.Errorf("target %s not verified after copy", dst)
|
||||
}
|
||||
if err := s3.Remove(ctx, src); err != nil {
|
||||
return fmt.Errorf("remove source: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func baseName(key string) string {
|
||||
if i := strings.LastIndex(key, "/"); i >= 0 {
|
||||
return key[i+1:]
|
||||
}
|
||||
return key
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"ganigeorgiev.com/pocketbase/pocketbase"
|
||||
"ganigeorgiev.com/pocketbase/pocketbase/core"
|
||||
"github.com/napnap75/archivist/internal/config"
|
||||
"github.com/napnap75/archivist/internal/pbapp"
|
||||
"github.com/napnap75/archivist/internal/storage"
|
||||
)
|
||||
|
||||
// fakeS3 is an in-memory bucket for the tool tests.
|
||||
type fakeS3 struct {
|
||||
objects map[string][]byte
|
||||
}
|
||||
|
||||
func (f *fakeS3) ListAll(ctx context.Context) ([]storage.InboxObject, error) {
|
||||
var keys []string
|
||||
for k := range f.objects {
|
||||
if strings.HasPrefix(k, "inbox/") {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := make([]storage.InboxObject, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
out = append(out, storage.InboxObject{Key: k, Size: int64(len(f.objects[k]))})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (f *fakeS3) Hash(ctx context.Context, key string) (string, error) {
|
||||
data, ok := f.objects[key]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("no such key %q", key)
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
return hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
func (f *fakeS3) Exists(ctx context.Context, key string) bool {
|
||||
_, ok := f.objects[key]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (f *fakeS3) Copy(ctx context.Context, srcKey, dstKey string) error {
|
||||
data, ok := f.objects[srcKey]
|
||||
if !ok {
|
||||
return fmt.Errorf("no such key %q", srcKey)
|
||||
}
|
||||
f.objects[dstKey] = data
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeS3) Remove(ctx context.Context, key string) error {
|
||||
delete(f.objects, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeS3) ResolveConflict(ctx context.Context, desired string) (string, error) {
|
||||
if !f.Exists(ctx, desired) {
|
||||
return desired, nil
|
||||
}
|
||||
for n := 1; n < 1000; n++ {
|
||||
ext := ""
|
||||
if i := strings.LastIndex(desired, "."); i >= 0 {
|
||||
ext = desired[i:]
|
||||
}
|
||||
candidate := fmt.Sprintf("%s (%d)%s", strings.TrimSuffix(desired, ext), n, ext)
|
||||
if !f.Exists(ctx, candidate) {
|
||||
return candidate, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("cannot resolve conflict on %q", desired)
|
||||
}
|
||||
|
||||
func newTestApp(t *testing.T) core.App {
|
||||
t.Helper()
|
||||
app := pocketbase.NewWithConfig(pocketbase.Config{DefaultDataDir: t.TempDir()})
|
||||
if err := app.Bootstrap(); err != nil {
|
||||
t.Fatalf("bootstrap PocketBase: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { app.ClearBootstrap() })
|
||||
cfg := &config.Config{
|
||||
Fields: map[string]config.Field{
|
||||
"documentType": {Name: "documentType", Type: "text"},
|
||||
},
|
||||
FieldList: []config.Field{{Name: "documentType", Type: "text"}},
|
||||
}
|
||||
if err := pbapp.Setup(app, "admin@example.net", "password-123456", cfg); err != nil {
|
||||
t.Fatalf("setup: %v", err)
|
||||
}
|
||||
return app
|
||||
}
|
||||
|
||||
func addDocument(t *testing.T, app core.App, sha, fileKey string) *core.Record {
|
||||
t.Helper()
|
||||
coll, err := app.FindCollectionByNameOrId("documents")
|
||||
if err != nil {
|
||||
t.Fatalf("find documents: %v", err)
|
||||
}
|
||||
rec := core.NewRecord(coll)
|
||||
rec.Set("sha256", sha)
|
||||
rec.Set("file_key", fileKey)
|
||||
rec.Set("status", "processed")
|
||||
if err := app.Save(rec); err != nil {
|
||||
t.Fatalf("save document: %v", err)
|
||||
}
|
||||
return rec
|
||||
}
|
||||
|
||||
func runScan(t *testing.T, app core.App, s3 S3Store) *Job {
|
||||
t.Helper()
|
||||
job := &Job{
|
||||
Tool: "find-lost-files",
|
||||
Status: StatusRunning,
|
||||
StartedAt: time.Now().UTC(),
|
||||
Summary: map[string]int{"scanned": 0, "ok": 0, "corrected": 0, "moved": 0, "duplicates": 0, "errors": 0},
|
||||
}
|
||||
if err := runFindLostFiles(context.Background(), app, s3, job); err != nil {
|
||||
t.Fatalf("runFindLostFiles: %v", err)
|
||||
}
|
||||
return job
|
||||
}
|
||||
|
||||
func TestFindLostFilesInPlace(t *testing.T) {
|
||||
app := newTestApp(t)
|
||||
s3 := &fakeS3{objects: map[string][]byte{"files/a.pdf": []byte("content-a")}}
|
||||
addDocument(t, app, mustHash("content-a"), "files/a.pdf")
|
||||
|
||||
job := runScan(t, app, s3)
|
||||
if job.Summary["ok"] != 1 || job.Summary["corrected"] != 0 || job.Summary["moved"] != 0 {
|
||||
t.Fatalf("summary = %v, want ok=1", job.Summary)
|
||||
}
|
||||
if len(job.Missing) != 0 {
|
||||
t.Fatalf("missing = %v, want empty", job.Missing)
|
||||
}
|
||||
if _, ok := s3.objects["files/a.pdf"]; !ok {
|
||||
t.Fatalf("object moved: %v", s3.objects)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindLostFilesRelocated(t *testing.T) {
|
||||
app := newTestApp(t)
|
||||
s3 := &fakeS3{objects: map[string][]byte{"files/new/a.pdf": []byte("content-a")}}
|
||||
rec := addDocument(t, app, mustHash("content-a"), "files/old/a.pdf")
|
||||
|
||||
job := runScan(t, app, s3)
|
||||
if job.Summary["corrected"] != 1 {
|
||||
t.Fatalf("summary = %v, want corrected=1", job.Summary)
|
||||
}
|
||||
fresh, err := app.FindRecordById("documents", rec.Id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fresh.GetString("file_key"); got != "files/new/a.pdf" {
|
||||
t.Fatalf("file_key = %q, want files/new/a.pdf", got)
|
||||
}
|
||||
events, err := app.FindAllRecords("document_events")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(events) != 1 {
|
||||
t.Fatalf("events = %d, want 1", len(events))
|
||||
}
|
||||
if events[0].GetString("event_type") != "relocated" {
|
||||
t.Fatalf("event_type = %q, want relocated", events[0].GetString("event_type"))
|
||||
}
|
||||
if _, ok := s3.objects["files/new/a.pdf"]; !ok {
|
||||
t.Fatalf("object should not move: %v", s3.objects)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindLostFilesUnknownGoesToInbox(t *testing.T) {
|
||||
app := newTestApp(t)
|
||||
s3 := &fakeS3{objects: map[string][]byte{"files/unknown.pdf": []byte("content-x")}}
|
||||
|
||||
job := runScan(t, app, s3)
|
||||
if job.Summary["moved"] != 1 {
|
||||
t.Fatalf("summary = %v, want moved=1", job.Summary)
|
||||
}
|
||||
if _, ok := s3.objects["inbox/unknown.pdf"]; !ok {
|
||||
t.Fatalf("object not in inbox: %v", s3.objects)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindLostFilesStrayDuplicateLeftInPlace(t *testing.T) {
|
||||
app := newTestApp(t)
|
||||
s3 := &fakeS3{objects: map[string][]byte{
|
||||
"files/orig.pdf": []byte("content-a"),
|
||||
"files/stray.pdf": []byte("content-a"),
|
||||
}}
|
||||
addDocument(t, app, mustHash("content-a"), "files/orig.pdf")
|
||||
|
||||
job := runScan(t, app, s3)
|
||||
if job.Summary["duplicates"] != 1 || job.Summary["ok"] != 1 {
|
||||
t.Fatalf("summary = %v, want duplicates=1 ok=1", job.Summary)
|
||||
}
|
||||
if _, ok := s3.objects["files/stray.pdf"]; !ok {
|
||||
t.Fatalf("stray copy should stay: %v", s3.objects)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindLostFilesMissingReported(t *testing.T) {
|
||||
app := newTestApp(t)
|
||||
s3 := &fakeS3{objects: map[string][]byte{}}
|
||||
addDocument(t, app, mustHash("content-a"), "files/gone.pdf")
|
||||
|
||||
job := runScan(t, app, s3)
|
||||
if len(job.Missing) != 1 || job.Missing[0] != "files/gone.pdf" {
|
||||
t.Fatalf("missing = %v, want [files/gone.pdf]", job.Missing)
|
||||
}
|
||||
if job.Summary["missing"] != 1 {
|
||||
t.Fatalf("summary = %v, want missing=1", job.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindLostFilesInboxExcluded(t *testing.T) {
|
||||
app := newTestApp(t)
|
||||
s3 := &fakeS3{objects: map[string][]byte{
|
||||
"inbox/upload.pdf": []byte("content-inbox"),
|
||||
"files/a.pdf": []byte("content-a"),
|
||||
}}
|
||||
addDocument(t, app, mustHash("content-a"), "files/a.pdf")
|
||||
addDocument(t, app, mustHash("content-inbox"), "inbox/upload.pdf")
|
||||
|
||||
job := runScan(t, app, s3)
|
||||
if job.Summary["scanned"] != 1 {
|
||||
t.Fatalf("summary = %v, want scanned=1 (inbox excluded)", job.Summary)
|
||||
}
|
||||
if len(job.Missing) != 0 {
|
||||
t.Fatalf("missing = %v, want empty (inbox records are the poller's business)", job.Missing)
|
||||
}
|
||||
if _, ok := s3.objects["inbox/upload.pdf"]; !ok {
|
||||
t.Fatalf("inbox object should stay: %v", s3.objects)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEventTypesIncludeRelocated(t *testing.T) {
|
||||
app := newTestApp(t)
|
||||
coll, err := app.FindCollectionByNameOrId("document_events")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
field, ok := coll.Fields.GetByName("event_type").(*core.SelectField)
|
||||
if !ok {
|
||||
t.Fatalf("event_type is not a select field")
|
||||
}
|
||||
for _, v := range []string{"imported", "corrected", "validated", "relocated"} {
|
||||
found := false
|
||||
for _, value := range field.Values {
|
||||
if value == v {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("event_type values %v miss %q", field.Values, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mustHash(content string) string {
|
||||
sum := sha256.Sum256([]byte(content))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
Reference in New Issue
Block a user