thomas/multiarch-docker-images
1
0
Fork 0
mirror of https://github.com/napnap75/multiarch-docker-images.git synced 2026-11-11 00:22:12 +01:00
Code Issues Projects Releases Wiki Activity

Corrections after first run

Browse Source
This commit is contained in:
thomas
2026-11-03 22:36:27 +01:00
parent 6f94674fc9
commit 02f7fd99bb
15 changed files with 268 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
archivist/.gitignore
+5
View File
@@ -0,0 +1,5 @@
.vscode/
pb_data/
tests/
__debug_bin*
go.sum
archivist/Dockerfile
+1
View File
@@ -40,3 +40,4 @@ USER 65532:65532
VOLUME /app/pb_data
EXPOSE 8090
ENTRYPOINT ["/archivist"]
CMD ["serve"]
archivist/README.md
+3 -1
View File
@@ -35,7 +35,7 @@ docker run -d --name archivist \
-e ARCHIVIST_S3_USE_SSL="true" \
-e ARCHIVIST_ADMIN_EMAIL="me@mydomain.fr" \
-e ARCHIVIST_ADMIN_PASSWORD="..." \
napnap75/archivist:latest
napnap75/archivist:latest serve --config /app/config.jsonc
```
The dashboard is available at `http://<server>:8090/`, the PocketBase admin at `http://<server>:8090/_/`.
@@ -72,6 +72,8 @@ The functional configuration is reloaded only when the container restarts.
| `ARCHIVIST_ADMIN_PASSWORD` | yes | Admin account password |
| `ARCHIVIST_HTTP_ADDR` | no | Listen address, default `0.0.0.0:8090` |
The PocketBase command is `serve` (the Docker image uses it by default). To load a different functional config file, pass `--config /path/to/config.jsonc` after `serve`. This overrides `ARCHIVIST_CONFIG`, which defaults to `config.jsonc`.
The `pb_data` volume holds the PocketBase SQLite database (documents, history, full-text search index): it must persist across restarts.
### The config.jsonc file
archivist/config.jsonc
+1 -1
View File
@@ -12,7 +12,7 @@
"fields": [
{ "name": "emittingCompany", "type": "text", "mandatory": true },
{ "name": "documentType", "type": "enum", "mandatory": true },
{ "name": "emissionDate", "type": "date", "mandatory": true },
{ "name": "emissionDate", "type": "date", "mandatory": true },
{ "name": "template", "type": "text", "mandatory": false },
{ "name": "bankAccount", "type": "enum", "mandatory": false },
{ "name": "address", "type": "text", "mandatory": false },
archivist/go.mod
+50 -6
View File
@@ -1,13 +1,57 @@
module github.com/napnap75/archivist
go 1.27
go 1.27.3
// Run `go mod tidy` to resolve the exact compatible versions.
require (
ganigeorgiev.com/pocketbase/dbx v1.13.0
ganigeorgiev.com/pocketbase/pocketbase v0.41.1
github.com/expr-lang/expr v1.17.6
github.com/ledongthuc/pdf v0.0.0-20240201144150-fd9d85c880aa
github.com/minio/minio-go/v7 v7.0.88
github.com/tailscale/hujson v0.4.0
github.com/expr-lang/expr v1.17.8
github.com/ledongthuc/pdf v0.0.0-20261020125853-5e46805f718e
github.com/minio/minio-go/v7 v7.4.1
github.com/spf13/pflag v1.0.10
github.com/tailscale/hujson v0.0.0-20260727124030-b80ff77dac4f
)
require (
ganigeorgiev.com/fexpr v0.7.0 // indirect
ganigeorgiev.com/pocketbase/ozzo-validation/v4 v4.4.1 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/disintegration/imaging v1.6.2 // indirect
github.com/domodwyer/mailyak/v3 v3.6.2 // indirect
github.com/dustin/go-humanize v1.1.0 // indirect
github.com/fatih/color v1.19.0 // indirect
github.com/fsnotify/fsnotify v1.10.1 // indirect
github.com/gabriel-vasile/mimetype v1.4.15 // indirect
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/klauspost/compress v1.19.2 // indirect
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/mattn/go-colorable v0.1.16 // indirect
github.com/mattn/go-isatty v0.0.24 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/ncruces/go-strftime v1.1.0 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/cobra v1.10.2 // indirect
github.com/tinylib/msgp v1.6.4 // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/crypto v0.58.0 // indirect
golang.org/x/image v0.47.0 // indirect
golang.org/x/net v0.62.0 // indirect
golang.org/x/oauth2 v0.37.0 // indirect
golang.org/x/sync v0.24.0 // indirect
golang.org/x/sys v0.49.0 // indirect
golang.org/x/text v0.43.0 // indirect
gopkg.in/ini.v1 v1.67.3 // indirect
modernc.org/libc v1.77.1 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.12.1 // indirect
modernc.org/sqlite v1.60.1 // indirect
)
archivist/internal/config/config.go
+41 -6
View File
@@ -12,6 +12,7 @@ import (
"time"
"github.com/expr-lang/expr"
"github.com/expr-lang/expr/vm"
"github.com/tailscale/hujson"
)
@@ -88,6 +89,7 @@ func Load(path string) (*Config, error) {
c.Fields[f.Name] = f
c.FieldList = append(c.FieldList, f)
}
SetKnownFields(c.FieldList)
// --- rules --------------------------------------------------------------
if err := validateRules(raw.Rules, c.Fields); err != nil {
@@ -124,7 +126,7 @@ func checkRuleTemplates(rules []*Rule, c *Config) error {
}
}
for _, child := range r.Rules {
if err := checkRuleTemplates(child, c); err != nil {
if err := checkRuleTemplates([]*Rule{child}, c); err != nil {
return err
}
}
@@ -166,8 +168,29 @@ func validateRules(rules []*Rule, fields map[string]Field) error {
}
// CompileExpr compiles one expression against the typed document context.
func CompileExpr(src string) (expr.Node, error) {
return expr.Compile(src, expr.Env(ExprEnv(nil, nil)))
func CompileExpr(src string) (*vm.Program, error) {
env := ExprEnv(nil, nil)
for _, name := range knownFields {
switch knownFieldTypes[name] {
case "number":
env[name] = float64(0)
case "date":
env[name] = time.Time{}
case "boolean":
env[name] = false
default:
env[name] = ""
}
}
opts := append([]expr.Option{expr.Env(env)}, exprOptions()...)
return expr.Compile(src, opts...)
}
func exprOptions() []expr.Option {
return []expr.Option{
expr.Function("firstMatch", FirstMatch),
expr.Function("period", Period),
}
}
// ExprEnv builds the evaluation environment: pages + declared fields.
@@ -186,11 +209,14 @@ func ExprEnv(pages []string, fields map[string]any) map[string]any {
}
var knownFields []string
var knownFieldTypes map[string]string
func SetKnownFields(fields []Field) {
knownFields = nil
knownFieldTypes = make(map[string]string, len(fields))
for _, f := range fields {
knownFields = append(knownFields, f.Name)
knownFieldTypes[f.Name] = f.Type
}
}
@@ -230,7 +256,12 @@ func (c *Config) Classify(pages []string) (map[string]any, []string) {
walk = func(rules []*Rule) {
for _, r := range rules {
env := ExprEnv(pages, fields)
res, err := expr.Eval(r.When, env)
program, err := CompileExpr(r.When)
if err != nil {
errs = append(errs, fmt.Sprintf("when: %v", err))
continue
}
res, err := expr.Run(program, env)
if err != nil {
errs = append(errs, fmt.Sprintf("when: %v", err))
continue
@@ -241,8 +272,12 @@ func (c *Config) Classify(pages []string) (map[string]any, []string) {
for _, s := range r.Set {
var v any
if s.Eval != "" {
var err error
v, err = expr.Eval(s.Eval, env)
program, err := CompileExpr(s.Eval)
if err != nil {
errs = append(errs, fmt.Sprintf("eval %s: %v", s.Field, err))
continue
}
v, err = expr.Run(program, env)
if err != nil {
errs = append(errs, fmt.Sprintf("eval %s: %v", s.Field, err))
continue
archivist/internal/config/config_test.go
+26
View File
@@ -0,0 +1,26 @@
package config
import (
"testing"
"time"
)
func TestDefaultConfigExpressionHelpers(t *testing.T) {
cfg, err := Load("../../config.jsonc")
if err != nil {
t.Fatalf("load default config: %v", err)
}
fields, rendered, errs := cfg.Evaluate([]string{
"Relevé de compte\nBRED\n12345678901\nDate : 15/01/2026",
})
if len(errs) != 0 {
t.Fatalf("evaluate default config: %v", errs)
}
if got, ok := fields["emissionDate"].(time.Time); !ok || got.Format("02/01/2006") != "15/01/2026" {
t.Fatalf("emissionDate = %v, want 15/01/2026", fields["emissionDate"])
}
if rendered == "" {
t.Fatal("default config rendered an empty file name")
}
}
archivist/internal/config/render.go
+2 -1
View File
@@ -8,13 +8,14 @@ import (
"time"
"github.com/expr-lang/expr"
"github.com/expr-lang/expr/vm"
)
var placeholderRe = regexp.MustCompile(`\{([^{}]+)\}`)
type tplPart struct {
literal string
program expr.Node
program *vm.Program
hasExpr bool
}
archivist/internal/pbapp/hooks.go
+8 -8
View File
@@ -5,28 +5,28 @@ import (
"fmt"
"time"
ganigeorgiev.com/pocketbase/pocketbase/core"
"ganigeorgiev.com/pocketbase/pocketbase/core"
"github.com/napnap75/archivist/internal/config"
"github.com/napnap75/archivist/internal/storage"
)
// BindHooks registers the request hooks (US-04, US-22, US-23).
func BindHooks(app *core.App, cfg *config.Config, s3 *storage.Client) {
func BindHooks(app core.App, cfg *config.Config, s3 *storage.Client) {
// US-04: document_events is append-only for everyone, superusers included.
app.OnRecordUpdateRequest(eventsName).PreBindFunc(func(e *core.RecordRequestEvent) error {
app.OnRecordUpdateRequest(eventsName).BindFunc(func(e *core.RecordRequestEvent) error {
return errAppendOnly
})
app.OnRecordDeleteRequest(eventsName).PreBindFunc(func(e *core.RecordRequestEvent) error {
app.OnRecordDeleteRequest(eventsName).BindFunc(func(e *core.RecordRequestEvent) error {
return errAppendOnly
})
// US-22 / US-23: corrections and validations through the standard UI.
app.OnRecordUpdateRequest("documents").PreBindFunc(func(e *core.RecordRequestEvent) error {
app.OnRecordUpdateRequest("documents").BindFunc(func(e *core.RecordRequestEvent) error {
return handleDocumentUpdate(app, cfg, s3, e)
})
}
func handleDocumentUpdate(app *core.App, cfg *config.Config, s3 *storage.Client, e *core.RecordRequestEvent) error {
func handleDocumentUpdate(app core.App, cfg *config.Config, s3 *storage.Client, e *core.RecordRequestEvent) error {
newRec := e.Record
// The original state, before the submitted data is persisted.
@@ -158,7 +158,7 @@ type Payload struct {
}
// AppendEvent appends one event to document_events.
func AppendEvent(app *core.App, documentId, eventType string, before, after Snapshot, errs []string) error {
func AppendEvent(app core.App, documentId, eventType string, before, after Snapshot, errs []string) error {
coll, err := app.FindCollectionByNameOrId(eventsName)
if err != nil {
return err
@@ -184,7 +184,7 @@ func AppendEvent(app *core.App, documentId, eventType string, before, after Snap
}
// AppendImportedEvent builds the imported event of US-16.
func AppendImportedEvent(app *core.App, documentId, fromKey, toKey string, valid bool, fields map[string]any, errs []string) error {
func AppendImportedEvent(app core.App, documentId, fromKey, toKey string, valid bool, fields map[string]any, errs []string) error {
before := Snapshot{"file_key": fromKey}
after := Snapshot{"file_key": toKey}
if valid {
archivist/internal/pbapp/setup.go
+11 -12
View File
@@ -6,14 +6,14 @@ import (
"errors"
"fmt"
ganigeorgiev.com/pocketbase/pocketbase/core"
"ganigeorgiev.com/pocketbase/pocketbase/core"
"github.com/napnap75/archivist/internal/config"
)
const eventsName = "document_events"
// Setup ensures the collections and the superuser account (US-03, US-04, US-01).
func Setup(app *core.App, adminEmail, adminPassword string, cfg *config.Config) error {
func Setup(app core.App, adminEmail, adminPassword string, cfg *config.Config) error {
if err := ensureDocuments(app, cfg); err != nil {
return err
}
@@ -23,11 +23,10 @@ func Setup(app *core.App, adminEmail, adminPassword string, cfg *config.Config)
if err := ensureSuperuser(app, adminEmail, adminPassword); err != nil {
return err
}
config.SetKnownFields(cfg.FieldList)
return nil
}
func ensureDocuments(app *core.App, cfg *config.Config) error {
func ensureDocuments(app core.App, cfg *config.Config) error {
coll, err := app.FindCollectionByNameOrId("documents")
if err != nil {
// First start: create the collection with its fixed columns (US-03).
@@ -59,9 +58,9 @@ func ensureDocuments(app *core.App, cfg *config.Config) error {
case "boolean":
newField = &core.BoolField{Name: f.Name}
case "enum":
// The vocabulary is an application notion (dashboard dropdown with
// free entry), not a schema constraint: empty allowed values.
newField = &core.SelectField{Name: f.Name, MaxSelect: 1}
// PocketBase select fields require a fixed non-empty vocabulary;
// config enums are free-form, so store them as text.
newField = &core.TextField{Name: f.Name}
}
if existing := coll.Fields.GetByName(f.Name); existing != nil {
if existing.Type() != newField.Type() {
@@ -79,9 +78,9 @@ func ensureDocuments(app *core.App, cfg *config.Config) error {
if f.GetSystem() {
continue
}
if _, declared := cfg.Fields[f.Name]; !declared && !isFixed(f.Name) {
app.Logger().Warn("documents: column removed", "field", f.Name, "reason", "removed from config")
coll.Fields.RemoveByName(f.Name)
if _, declared := cfg.Fields[f.GetName()]; !declared && !isFixed(f.GetName()) {
app.Logger().Warn("documents: column removed", "field", f.GetName(), "reason", "removed from config")
coll.Fields.RemoveByName(f.GetName())
}
}
@@ -104,7 +103,7 @@ func isFixed(name string) bool {
return false
}
func ensureEvents(app *core.App) error {
func ensureEvents(app core.App) error {
if _, err := app.FindCollectionByNameOrId(eventsName); err == nil {
return nil
}
@@ -128,7 +127,7 @@ func ensureEvents(app *core.App) error {
return nil
}
func ensureSuperuser(app *core.App, email, password string) error {
func ensureSuperuser(app core.App, email, password string) error {
superusers, err := app.FindCollectionByNameOrId(core.CollectionNameSuperusers)
if err != nil {
return err
archivist/internal/pbapp/setup_test.go
+42
View File
@@ -0,0 +1,42 @@
package pbapp
import (
"testing"
"ganigeorgiev.com/pocketbase/pocketbase"
"github.com/napnap75/archivist/internal/config"
)
func TestEnsureDocumentsUsesTextForFreeFormEnums(t *testing.T) {
app := pocketbase.NewWithConfig(pocketbase.Config{DefaultDataDir: t.TempDir()})
if err := app.Bootstrap(); err != nil {
t.Fatalf("bootstrap PocketBase: %v", err)
}
defer app.ClearBootstrap()
fields := []config.Field{
{Name: "documentType", Type: "enum"},
{Name: "bankAccount", Type: "enum"},
}
cfg := &config.Config{
Fields: map[string]config.Field{
"documentType": fields[0],
"bankAccount": fields[1],
},
FieldList: fields,
}
if err := ensureDocuments(app, cfg); err != nil {
t.Fatalf("ensure documents: %v", err)
}
coll, err := app.FindCollectionByNameOrId("documents")
if err != nil {
t.Fatalf("find documents collection: %v", err)
}
for _, name := range []string{"documentType", "bankAccount"} {
field := coll.Fields.GetByName(name)
if field == nil || field.Type() != "text" {
t.Errorf("field %q = %v, want text field", name, field)
}
}
}
archivist/internal/pipeline/pipeline.go
+10 -6
View File
@@ -8,8 +8,8 @@ import (
"fmt"
"strings"
ganigeorgiev.com/pocketbase/dbx"
ganigeorgiev.com/pocketbase/pocketbase/core"
"ganigeorgiev.com/pocketbase/dbx"
"ganigeorgiev.com/pocketbase/pocketbase/core"
"github.com/ledongthuc/pdf"
"github.com/napnap75/archivist/internal/config"
"github.com/napnap75/archivist/internal/pbapp"
@@ -24,7 +24,7 @@ type pollState struct {
var stable = map[string]pollState{}
func RunPoll(app *core.App, cfg *config.Config, s3 *storage.Client) {
func RunPoll(app core.App, cfg *config.Config, s3 *storage.Client) {
ctx := context.Background()
// US-11: recover interrupted filings first.
@@ -54,7 +54,7 @@ func RunPoll(app *core.App, cfg *config.Config, s3 *storage.Client) {
}
}
func processOne(app *core.App, cfg *config.Config, s3 *storage.Client, ctx context.Context, key string) error {
func processOne(app core.App, cfg *config.Config, s3 *storage.Client, ctx context.Context, key string) error {
// US-12: download in memory, sha256 on the fly.
data, hash, err := s3.Get(ctx, key)
if err != nil {
@@ -154,7 +154,7 @@ func fileDocument(s3 *storage.Client, ctx context.Context, record *core.Record,
// recoverStuck completes interrupted filings: records whose file_key is still
// under inbox/ (US-11).
func recoverStuck(app *core.App, cfg *config.Config, s3 *storage.Client, ctx context.Context) {
func recoverStuck(app core.App, cfg *config.Config, s3 *storage.Client, ctx context.Context) {
stuck, err := app.FindRecordsByFilter("documents", "file_key ~ {:p}", "", 100, 0, dbx.Params{"p": "inbox/"})
if err != nil {
app.Logger().Error("recovery: query failed", "error", err)
@@ -213,7 +213,11 @@ func extractPages(data []byte) ([]string, error) {
if p.V.IsNull() {
return nil, fmt.Errorf("page %d is null", i)
}
pages = append(pages, p.GetPlainText(nil))
text, err := p.GetPlainText(nil)
if err != nil {
return nil, fmt.Errorf("page %d: %w", i, err)
}
pages = append(pages, text)
}
if len(pages) == 0 {
return nil, fmt.Errorf("no page extracted")
archivist/internal/storage/s3.go
+1 -1
View File
@@ -97,7 +97,7 @@ func (c *Client) Exists(ctx context.Context, key string) bool {
func (c *Client) Copy(ctx context.Context, srcKey, dstKey string) error {
_, err := c.mc.CopyObject(ctx,
minio.CopyDestOptions{Bucket: c.Bucket, Object: dstKey},
minio.CopySourceOptions{Bucket: c.Bucket, Object: srcKey})
minio.CopySrcOptions{Bucket: c.Bucket, Object: srcKey})
return err
}
archivist/main.go
+24 -4
View File
@@ -4,12 +4,13 @@ import (
"log"
"os"
ganigeorgiev.com/pocketbase/pocketbase"
ganigeorgiev.com/pocketbase/pocketbase/core"
"ganigeorgiev.com/pocketbase/pocketbase"
"ganigeorgiev.com/pocketbase/pocketbase/core"
"github.com/napnap75/archivist/internal/config"
"github.com/napnap75/archivist/internal/pbapp"
"github.com/napnap75/archivist/internal/pipeline"
"github.com/napnap75/archivist/internal/storage"
"github.com/spf13/pflag"
)
// EnvConfig is the technical configuration, from environment variables (US-01).
@@ -64,11 +65,27 @@ func LoadEnvConfig() (*EnvConfig, error) {
return c, nil
}
func configPathFromArgs(args []string, defaultPath string) (string, error) {
var configPath string
flags := pflag.NewFlagSet("archivist", pflag.ContinueOnError)
flags.SetInterspersed(true)
flags.ParseErrorsWhitelist.UnknownFlags = true
flags.StringVar(&configPath, "config", defaultPath, "Path to the functional JSONC configuration")
if err := flags.Parse(args); err != nil {
return "", err
}
return configPath, nil
}
func main() {
envCfg, err := LoadEnvConfig()
if err != nil {
log.Fatal(err)
}
envCfg.ConfigPath, err = configPathFromArgs(os.Args[1:], envCfg.ConfigPath)
if err != nil {
log.Fatalf("invalid command line: %v", err)
}
// US-02: load and validate the functional configuration. Full rejection,
// the process does not continue with a partial configuration (US-05).
@@ -86,6 +103,7 @@ func main() {
}
app := pocketbase.New()
app.RootCmd.PersistentFlags().StringVar(&envCfg.ConfigPath, "config", envCfg.ConfigPath, "Path to the functional JSONC configuration")
// The PocketBase logger and database are available only after bootstrap;
// everything logged before this point goes to stderr (arch.md section 9).
if err := app.Bootstrap(); err != nil {
@@ -119,9 +137,11 @@ func main() {
return e.Next()
})
// US-10..US-17: the poller, on the configured cron.
app.CronAdd(envCfg.PollCron, "archivist-poller", func() {
if err := app.Cron().Add("archivist-poller", envCfg.PollCron, func() {
pipeline.RunPoll(app, cfg, s3)
})
}); err != nil {
log.Fatalf("register archivist poller: %v", err)
}
app.RootCmd.PersistentFlags().Set("http", envCfg.HTTPAddr)
if err := app.Start(); err != nil {
archivist/main_test.go
+43
View File
@@ -0,0 +1,43 @@
package main
import "testing"
func TestConfigPathFromArgs(t *testing.T) {
tests := []struct {
name string
args []string
defaultPath string
want string
}{
{
name: "default path",
args: []string{"serve"},
defaultPath: "config.jsonc",
want: "config.jsonc",
},
{
name: "config after serve",
args: []string{"serve", "--config", "/etc/archivist/custom.jsonc"},
defaultPath: "config.jsonc",
want: "/etc/archivist/custom.jsonc",
},
{
name: "config among PocketBase flags",
args: []string{"serve", "--http=127.0.0.1:8090", "--config=/etc/archivist/custom.jsonc"},
defaultPath: "config.jsonc",
want: "/etc/archivist/custom.jsonc",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := configPathFromArgs(tt.args, tt.defaultPath)
if err != nil {
t.Fatalf("configPathFromArgs() error = %v", err)
}
if got != tt.want {
t.Errorf("configPathFromArgs() = %q, want %q", got, tt.want)
}
})
}
}