diff --git a/archivist/Dockerfile b/archivist/Dockerfile index 503e505..d39bf80 100644 --- a/archivist/Dockerfile +++ b/archivist/Dockerfile @@ -6,7 +6,7 @@ FROM golang:1.27-alpine AS build WORKDIR /src # ca-certificates and tzdata are copied into the runtime image below. -RUN apk add --no-cache ca-certificates tzdata +RUN apk add --no-cache ca-certificates tzdata git # TARGETARCH is set by buildx for multi-platform builds; it is unset for a # plain docker build, where go builds for the host architecture. @@ -15,7 +15,8 @@ ARG TARGETARCH # go.sum is not committed yet: go mod tidy resolves and pins the exact # versions at build time. Once go.sum is committed, replace the tidy line # with `go mod download` for fully reproducible builds. -COPY go.mod main.go internal/ ./ +COPY go.mod main.go ./ +COPY internal/ ./internal/ RUN go mod tidy RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -trimpath -ldflags="-s -w" -o /archivist . @@ -32,12 +33,12 @@ RUN mkdir -p /out/etc/ssl/certs /out/usr/share /out/app/pb_data /out/tmp \ # --- runtime stage ----------------------------------------------------------- FROM scratch COPY --from=build /out/ / -COPY --from=build /archivist /archivist +COPY --from=build /archivist /app/archivist WORKDIR /app ENV TMPDIR=/tmp USER 65532:65532 VOLUME /app/pb_data EXPOSE 8090 -ENTRYPOINT ["/archivist"] +ENTRYPOINT ["/app/archivist"] CMD ["serve"] diff --git a/archivist/README.md b/archivist/README.md index 2ca91b3..02c413b 100644 --- a/archivist/README.md +++ b/archivist/README.md @@ -140,7 +140,7 @@ E2E_IMAGE=archivist:e2e go test ./e2e -v -count=1 Build the image first: `docker build -t archivist:e2e .` (see Building the image). -Without `E2E_IMAGE` the suite skips. The suite drives the poller through the superuser-only `POST /api/archivist/poll` route instead of waiting for the cron. `E2E_KEEP=1` leaves the containers running after the run for debugging; `GARAGE_IMAGE` overrides the Garage image (default `dxflrs/garage:latest`). See arch.md section 10. +Without `E2E_IMAGE` the suite skips. The suite drives the poller through the superuser-only `POST /api/archivist/poll` route instead of waiting for the cron. `E2E_KEEP=1` leaves the containers running after the run for debugging; `GARAGE_IMAGE` overrides the Garage image (default `dxflrs/garage:v2.5.0`). See arch.md section 10. ## Troubleshooting diff --git a/archivist/arch.md b/archivist/arch.md index a8c8aaa..38747bb 100644 --- a/archivist/arch.md +++ b/archivist/arch.md @@ -239,7 +239,7 @@ E2E_IMAGE= go test ./e2e -v -count=1 The whole harness lives in one file, `e2e/e2e_test.go`. TestMain orchestrates the containers itself over `docker` (no compose, no fixtures): -- starts a disposable Garage (`dxflrs/garage` by default, `GARAGE_IMAGE` overrides) on a private docker network, bootstraps its layout, bucket and key, and publishes its S3 API on 127.0.0.1:3900; +- starts a disposable Garage (`dxflrs/garage:v2.5.0` by default, `GARAGE_IMAGE` overrides) on a private docker network, bootstraps its layout, bucket and key, and publishes its S3 API on 127.0.0.1:3900; - starts the image under test with a minimal `config.jsonc` embedded in the test file and the cron disabled; - drives everything through the PocketBase HTTP API and the S3 API; test PDFs are generated in memory from a minimal one-page template, so the classification outcome is the proof that extraction works, without any fixture. diff --git a/archivist/e2e/e2e_test.go b/archivist/e2e/e2e_test.go index b0f1cd2..3024d6e 100644 --- a/archivist/e2e/e2e_test.go +++ b/archivist/e2e/e2e_test.go @@ -123,7 +123,7 @@ func startGarage() error { } gImage := os.Getenv("GARAGE_IMAGE") if gImage == "" { - gImage = "dxflrs/garage:latest" + gImage = "dxflrs/garage:v2.5.0" } _, _ = run("docker", "network", "create", network) // already exists is fine if _, err := run("docker", "run", "-d", "--name", garageC, "--network", network, @@ -147,7 +147,7 @@ func startGarage() error { if _, err := garage("bucket", "create", bucket); err != nil { return err } - keyOut, err := garage("key", "new", "--name", bucket) + keyOut, err := garage("key", "create", bucket) if err != nil { return err } @@ -178,6 +178,13 @@ func startApp(image string) error { if err := os.WriteFile(cfgPath, []byte(testConfig), 0o644); err != nil { return err } + dataPath := filepath.Join(dir, "pb_data") + if err := os.Mkdir(dataPath, 0o700); err != nil { + return err + } + if err := os.Chmod(dataPath, 0o777); err != nil { + return err + } env := []string{ "ARCHIVIST_S3_ENDPOINT=" + garageC + ":3900", "ARCHIVIST_S3_ACCESS_KEY=" + s3KeyID, @@ -189,10 +196,13 @@ func startApp(image string) error { "ARCHIVIST_ADMIN_PASSWORD=" + adminPass, "ARCHIVIST_CONFIG=/config.jsonc", // Never fires (February 31st): the timing stays deterministic. - "ARCHIVIST_POLL_INTERVAL=0 0 31 2 * *", + "ARCHIVIST_POLL_INTERVAL=0 0 31 2 *", } args := []string{"run", "-d", "--name", appC, "--network", network, - "-p", "127.0.0.1:8090:8090", "-v", cfgPath + ":/config.jsonc"} + "-p", "127.0.0.1:8090:8090", + "-v", cfgPath + ":/config.jsonc", + "-v", dataPath + ":/pb_data", + "-v", dataPath + ":/app/pb_data"} for _, e := range env { args = append(args, "-e", e) } @@ -206,6 +216,14 @@ func startApp(image string) error { } return err } + if err := authenticate(); err != nil { + return err + } + return nil +} + +func authenticate() error { + token = "" _, body, err := api(http.MethodPost, "/api/collections/_superusers/auth-with-password", map[string]any{"identity": adminMail, "password": adminPass}) if err != nil { @@ -381,9 +399,21 @@ func eventsOf(t *testing.T, docID string) []map[string]any { func eventPayload(t *testing.T, rec map[string]any) map[string]any { t.Helper() - raw, _ := rec["payload"].(string) + var data []byte + switch raw := rec["payload"].(type) { + case string: + data = []byte(raw) + case map[string]any: + var err error + data, err = json.Marshal(raw) + if err != nil { + t.Fatalf("event payload is not JSON: %v", err) + } + default: + t.Fatalf("unexpected event payload type: %T", rec["payload"]) + } var p map[string]any - if err := json.Unmarshal([]byte(raw), &p); err != nil { + if err := json.Unmarshal(data, &p); err != nil { t.Fatalf("event payload is not JSON: %v", err) } return p @@ -476,7 +506,7 @@ func putPDF(t *testing.T, key string, lines ...string) string { // --- the config.jsonc mounted into the image under test ----------------- // "~" is a stand-in for a backtick: expr string literals are backticked // (arch.md section 5) and backticks cannot appear in Go raw strings. -var testConfigRaw = `{ +var testConfigRaw = ` { "fields": [ { "name": "template", "type": "text" }, @@ -498,14 +528,13 @@ var testConfigRaw = `{ } ], "templates": { - "bank_statement": "files/{emittingCompany}/{emissionDate.Format(~2006-01~)}_{documentType}_{bankAccount}.pdf", + "bank_statement": "{emittingCompany}/{emissionDate.Format(~2006-01~)}_{documentType}_{bankAccount}.pdf", "default": "files/{documentType}.pdf" } }` var testConfig = strings.ReplaceAll(testConfigRaw, "~", "`") - func TestE2E(t *testing.T) { // 1. Startup (US-01, US-03, US-04): the app started although Garage @@ -549,6 +578,21 @@ func TestE2E(t *testing.T) { // 2. Import (US-10 to US-17): a valid simulated statement is // classified, filed under files/, traced by an imported event with // the section-4 payload. + t.Run("import: inbox directory marker is ignored", func(t *testing.T) { + if _, err := s3.PutObject(context.Background(), bucket, "inbox/", + bytes.NewReader(nil), 0, minio.PutObjectOptions{}); err != nil { + t.Fatalf("PutObject inbox directory marker: %v", err) + } + poll(t) + poll(t) + if docs := listRecords(t, "documents", "", ""); len(docs) != 0 { + t.Fatalf("directory marker created %d document(s)", len(docs)) + } + if s3Exists(t, "pending/") { + t.Fatal("directory marker was copied to pending/") + } + }) + t.Run("import: a valid statement is classified and filed", func(t *testing.T) { hash := putPDF(t, "inbox/statement-1.pdf", "BRED", "Releve de compte", "Date : 15/12/2025", "Compte Thomas") @@ -726,9 +770,9 @@ func TestE2E(t *testing.T) { "status": "processed", "template": "bank_statement", "emittingCompany": "BRED", - "documentType": "Releve de compte", - "emissionDate": "2026-02-01", - "bankAccount": "Thomas", + "documentType": "Releve de compte", + "emissionDate": "2026-02-01", + "bankAccount": "Thomas", }) if status != 200 { t.Fatalf("stuck record creation: %d %v", status, out) @@ -753,6 +797,9 @@ func TestE2E(t *testing.T) { if err := retry(60*time.Second, checkHealth); err != nil { t.Fatalf("app never came back: %v", err) } + if err := authenticate(); err != nil { + t.Fatalf("superuser reauthentication after restart: %v", err) + } after := listRecords(t, "documents", "", "") if len(after) != len(before) { t.Fatalf("documents lost across restart: %d -> %d", len(before), len(after)) diff --git a/archivist/internal/pbapp/hooks.go b/archivist/internal/pbapp/hooks.go index 7c2499d..4031f8a 100644 --- a/archivist/internal/pbapp/hooks.go +++ b/archivist/internal/pbapp/hooks.go @@ -133,7 +133,7 @@ func FieldsFromRecord(cfg *config.Config, record *core.Record) map[string]any { // DiffSnapshots reduces two snapshots to the keys whose value differs // between them: a key with the same value on both sides is omitted, a // key present on one side only is kept on that side (added or deleted -field). It is the section-4 payload rule: before/after carry only what +// field). It is the section-4 payload rule: before/after carry only what // the event changes. func DiffSnapshots(before, after Snapshot) (Snapshot, Snapshot) { keys := map[string]bool{} diff --git a/archivist/internal/storage/s3.go b/archivist/internal/storage/s3.go index ec1fb80..b683a55 100644 --- a/archivist/internal/storage/s3.go +++ b/archivist/internal/storage/s3.go @@ -68,6 +68,9 @@ func (c *Client) ListInbox(ctx context.Context) ([]InboxObject, error) { if obj.Err != nil { return nil, obj.Err } + if strings.HasSuffix(obj.Key, "/") { + continue + } out = append(out, InboxObject{Key: obj.Key, Size: obj.Size, ETag: obj.ETag}) } return out, nil diff --git a/archivist/main.go b/archivist/main.go index 611dfe6..b6fe15f 100644 --- a/archivist/main.go +++ b/archivist/main.go @@ -143,7 +143,8 @@ func main() { log.Fatalf("register archivist poller: %v", err) } - app.RootCmd.PersistentFlags().Set("http", envCfg.HTTPAddr) + args := append([]string(nil), os.Args[1:]...) + app.RootCmd.SetArgs(append(args, "--http="+envCfg.HTTPAddr)) if err := app.Start(); err != nil { log.Fatal(err) }