thomas/multiarch-docker-images
1
0
Fork 0
mirror of https://github.com/napnap75/multiarch-docker-images.git synced 2026-11-12 00:52:11 +01:00
Code Issues Projects Releases Wiki Activity

Corrections after first run

Browse Source
This commit is contained in:
thomas
2026-11-03 22:36:27 +01:00
parent 6f94674fc9
commit 02f7fd99bb
15 changed files with 268 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
archivist/internal/config/config.go
+41 -6
View File
@@ -12,6 +12,7 @@ import (
"time"
"github.com/expr-lang/expr"
"github.com/expr-lang/expr/vm"
"github.com/tailscale/hujson"
)
@@ -88,6 +89,7 @@ func Load(path string) (*Config, error) {
c.Fields[f.Name] = f
c.FieldList = append(c.FieldList, f)
}
SetKnownFields(c.FieldList)
// --- rules --------------------------------------------------------------
if err := validateRules(raw.Rules, c.Fields); err != nil {
@@ -124,7 +126,7 @@ func checkRuleTemplates(rules []*Rule, c *Config) error {
}
}
for _, child := range r.Rules {
if err := checkRuleTemplates(child, c); err != nil {
if err := checkRuleTemplates([]*Rule{child}, c); err != nil {
return err
}
}
@@ -166,8 +168,29 @@ func validateRules(rules []*Rule, fields map[string]Field) error {
}
// CompileExpr compiles one expression against the typed document context.
func CompileExpr(src string) (expr.Node, error) {
return expr.Compile(src, expr.Env(ExprEnv(nil, nil)))
func CompileExpr(src string) (*vm.Program, error) {
env := ExprEnv(nil, nil)
for _, name := range knownFields {
switch knownFieldTypes[name] {
case "number":
env[name] = float64(0)
case "date":
env[name] = time.Time{}
case "boolean":
env[name] = false
default:
env[name] = ""
}
}
opts := append([]expr.Option{expr.Env(env)}, exprOptions()...)
return expr.Compile(src, opts...)
}
func exprOptions() []expr.Option {
return []expr.Option{
expr.Function("firstMatch", FirstMatch),
expr.Function("period", Period),
}
}
// ExprEnv builds the evaluation environment: pages + declared fields.
@@ -186,11 +209,14 @@ func ExprEnv(pages []string, fields map[string]any) map[string]any {
}
var knownFields []string
var knownFieldTypes map[string]string
func SetKnownFields(fields []Field) {
knownFields = nil
knownFieldTypes = make(map[string]string, len(fields))
for _, f := range fields {
knownFields = append(knownFields, f.Name)
knownFieldTypes[f.Name] = f.Type
}
}
@@ -230,7 +256,12 @@ func (c *Config) Classify(pages []string) (map[string]any, []string) {
walk = func(rules []*Rule) {
for _, r := range rules {
env := ExprEnv(pages, fields)
res, err := expr.Eval(r.When, env)
program, err := CompileExpr(r.When)
if err != nil {
errs = append(errs, fmt.Sprintf("when: %v", err))
continue
}
res, err := expr.Run(program, env)
if err != nil {
errs = append(errs, fmt.Sprintf("when: %v", err))
continue
@@ -241,8 +272,12 @@ func (c *Config) Classify(pages []string) (map[string]any, []string) {
for _, s := range r.Set {
var v any
if s.Eval != "" {
var err error
v, err = expr.Eval(s.Eval, env)
program, err := CompileExpr(s.Eval)
if err != nil {
errs = append(errs, fmt.Sprintf("eval %s: %v", s.Field, err))
continue
}
v, err = expr.Run(program, env)
if err != nil {
errs = append(errs, fmt.Sprintf("eval %s: %v", s.Field, err))
continue
archivist/internal/config/config_test.go
+26
View File
@@ -0,0 +1,26 @@
package config
import (
"testing"
"time"
)
func TestDefaultConfigExpressionHelpers(t *testing.T) {
cfg, err := Load("../../config.jsonc")
if err != nil {
t.Fatalf("load default config: %v", err)
}
fields, rendered, errs := cfg.Evaluate([]string{
"Relevé de compte\nBRED\n12345678901\nDate : 15/01/2026",
})
if len(errs) != 0 {
t.Fatalf("evaluate default config: %v", errs)
}
if got, ok := fields["emissionDate"].(time.Time); !ok || got.Format("02/01/2006") != "15/01/2026" {
t.Fatalf("emissionDate = %v, want 15/01/2026", fields["emissionDate"])
}
if rendered == "" {
t.Fatal("default config rendered an empty file name")
}
}
archivist/internal/config/render.go
+2 -1
View File
@@ -8,13 +8,14 @@ import (
"time"
"github.com/expr-lang/expr"
"github.com/expr-lang/expr/vm"
)
var placeholderRe = regexp.MustCompile(`\{([^{}]+)\}`)
type tplPart struct {
literal string
program expr.Node
program *vm.Program
hasExpr bool
}
archivist/internal/pbapp/hooks.go
+8 -8
View File
@@ -5,28 +5,28 @@ import (
"fmt"
"time"
ganigeorgiev.com/pocketbase/pocketbase/core"
"ganigeorgiev.com/pocketbase/pocketbase/core"
"github.com/napnap75/archivist/internal/config"
"github.com/napnap75/archivist/internal/storage"
)
// BindHooks registers the request hooks (US-04, US-22, US-23).
func BindHooks(app *core.App, cfg *config.Config, s3 *storage.Client) {
func BindHooks(app core.App, cfg *config.Config, s3 *storage.Client) {
// US-04: document_events is append-only for everyone, superusers included.
app.OnRecordUpdateRequest(eventsName).PreBindFunc(func(e *core.RecordRequestEvent) error {
app.OnRecordUpdateRequest(eventsName).BindFunc(func(e *core.RecordRequestEvent) error {
return errAppendOnly
})
app.OnRecordDeleteRequest(eventsName).PreBindFunc(func(e *core.RecordRequestEvent) error {
app.OnRecordDeleteRequest(eventsName).BindFunc(func(e *core.RecordRequestEvent) error {
return errAppendOnly
})
// US-22 / US-23: corrections and validations through the standard UI.
app.OnRecordUpdateRequest("documents").PreBindFunc(func(e *core.RecordRequestEvent) error {
app.OnRecordUpdateRequest("documents").BindFunc(func(e *core.RecordRequestEvent) error {
return handleDocumentUpdate(app, cfg, s3, e)
})
}
func handleDocumentUpdate(app *core.App, cfg *config.Config, s3 *storage.Client, e *core.RecordRequestEvent) error {
func handleDocumentUpdate(app core.App, cfg *config.Config, s3 *storage.Client, e *core.RecordRequestEvent) error {
newRec := e.Record
// The original state, before the submitted data is persisted.
@@ -158,7 +158,7 @@ type Payload struct {
}
// AppendEvent appends one event to document_events.
func AppendEvent(app *core.App, documentId, eventType string, before, after Snapshot, errs []string) error {
func AppendEvent(app core.App, documentId, eventType string, before, after Snapshot, errs []string) error {
coll, err := app.FindCollectionByNameOrId(eventsName)
if err != nil {
return err
@@ -184,7 +184,7 @@ func AppendEvent(app *core.App, documentId, eventType string, before, after Snap
}
// AppendImportedEvent builds the imported event of US-16.
func AppendImportedEvent(app *core.App, documentId, fromKey, toKey string, valid bool, fields map[string]any, errs []string) error {
func AppendImportedEvent(app core.App, documentId, fromKey, toKey string, valid bool, fields map[string]any, errs []string) error {
before := Snapshot{"file_key": fromKey}
after := Snapshot{"file_key": toKey}
if valid {
archivist/internal/pbapp/setup.go
+11 -12
View File
@@ -6,14 +6,14 @@ import (
"errors"
"fmt"
ganigeorgiev.com/pocketbase/pocketbase/core"
"ganigeorgiev.com/pocketbase/pocketbase/core"
"github.com/napnap75/archivist/internal/config"
)
const eventsName = "document_events"
// Setup ensures the collections and the superuser account (US-03, US-04, US-01).
func Setup(app *core.App, adminEmail, adminPassword string, cfg *config.Config) error {
func Setup(app core.App, adminEmail, adminPassword string, cfg *config.Config) error {
if err := ensureDocuments(app, cfg); err != nil {
return err
}
@@ -23,11 +23,10 @@ func Setup(app *core.App, adminEmail, adminPassword string, cfg *config.Config)
if err := ensureSuperuser(app, adminEmail, adminPassword); err != nil {
return err
}
config.SetKnownFields(cfg.FieldList)
return nil
}
func ensureDocuments(app *core.App, cfg *config.Config) error {
func ensureDocuments(app core.App, cfg *config.Config) error {
coll, err := app.FindCollectionByNameOrId("documents")
if err != nil {
// First start: create the collection with its fixed columns (US-03).
@@ -59,9 +58,9 @@ func ensureDocuments(app *core.App, cfg *config.Config) error {
case "boolean":
newField = &core.BoolField{Name: f.Name}
case "enum":
// The vocabulary is an application notion (dashboard dropdown with
// free entry), not a schema constraint: empty allowed values.
newField = &core.SelectField{Name: f.Name, MaxSelect: 1}
// PocketBase select fields require a fixed non-empty vocabulary;
// config enums are free-form, so store them as text.
newField = &core.TextField{Name: f.Name}
}
if existing := coll.Fields.GetByName(f.Name); existing != nil {
if existing.Type() != newField.Type() {
@@ -79,9 +78,9 @@ func ensureDocuments(app *core.App, cfg *config.Config) error {
if f.GetSystem() {
continue
}
if _, declared := cfg.Fields[f.Name]; !declared && !isFixed(f.Name) {
app.Logger().Warn("documents: column removed", "field", f.Name, "reason", "removed from config")
coll.Fields.RemoveByName(f.Name)
if _, declared := cfg.Fields[f.GetName()]; !declared && !isFixed(f.GetName()) {
app.Logger().Warn("documents: column removed", "field", f.GetName(), "reason", "removed from config")
coll.Fields.RemoveByName(f.GetName())
}
}
@@ -104,7 +103,7 @@ func isFixed(name string) bool {
return false
}
func ensureEvents(app *core.App) error {
func ensureEvents(app core.App) error {
if _, err := app.FindCollectionByNameOrId(eventsName); err == nil {
return nil
}
@@ -128,7 +127,7 @@ func ensureEvents(app *core.App) error {
return nil
}
func ensureSuperuser(app *core.App, email, password string) error {
func ensureSuperuser(app core.App, email, password string) error {
superusers, err := app.FindCollectionByNameOrId(core.CollectionNameSuperusers)
if err != nil {
return err
archivist/internal/pbapp/setup_test.go
+42
View File
@@ -0,0 +1,42 @@
package pbapp
import (
"testing"
"ganigeorgiev.com/pocketbase/pocketbase"
"github.com/napnap75/archivist/internal/config"
)
func TestEnsureDocumentsUsesTextForFreeFormEnums(t *testing.T) {
app := pocketbase.NewWithConfig(pocketbase.Config{DefaultDataDir: t.TempDir()})
if err := app.Bootstrap(); err != nil {
t.Fatalf("bootstrap PocketBase: %v", err)
}
defer app.ClearBootstrap()
fields := []config.Field{
{Name: "documentType", Type: "enum"},
{Name: "bankAccount", Type: "enum"},
}
cfg := &config.Config{
Fields: map[string]config.Field{
"documentType": fields[0],
"bankAccount": fields[1],
},
FieldList: fields,
}
if err := ensureDocuments(app, cfg); err != nil {
t.Fatalf("ensure documents: %v", err)
}
coll, err := app.FindCollectionByNameOrId("documents")
if err != nil {
t.Fatalf("find documents collection: %v", err)
}
for _, name := range []string{"documentType", "bankAccount"} {
field := coll.Fields.GetByName(name)
if field == nil || field.Type() != "text" {
t.Errorf("field %q = %v, want text field", name, field)
}
}
}
archivist/internal/pipeline/pipeline.go
+10 -6
View File
@@ -8,8 +8,8 @@ import (
"fmt"
"strings"
ganigeorgiev.com/pocketbase/dbx"
ganigeorgiev.com/pocketbase/pocketbase/core"
"ganigeorgiev.com/pocketbase/dbx"
"ganigeorgiev.com/pocketbase/pocketbase/core"
"github.com/ledongthuc/pdf"
"github.com/napnap75/archivist/internal/config"
"github.com/napnap75/archivist/internal/pbapp"
@@ -24,7 +24,7 @@ type pollState struct {
var stable = map[string]pollState{}
func RunPoll(app *core.App, cfg *config.Config, s3 *storage.Client) {
func RunPoll(app core.App, cfg *config.Config, s3 *storage.Client) {
ctx := context.Background()
// US-11: recover interrupted filings first.
@@ -54,7 +54,7 @@ func RunPoll(app *core.App, cfg *config.Config, s3 *storage.Client) {
}
}
func processOne(app *core.App, cfg *config.Config, s3 *storage.Client, ctx context.Context, key string) error {
func processOne(app core.App, cfg *config.Config, s3 *storage.Client, ctx context.Context, key string) error {
// US-12: download in memory, sha256 on the fly.
data, hash, err := s3.Get(ctx, key)
if err != nil {
@@ -154,7 +154,7 @@ func fileDocument(s3 *storage.Client, ctx context.Context, record *core.Record,
// recoverStuck completes interrupted filings: records whose file_key is still
// under inbox/ (US-11).
func recoverStuck(app *core.App, cfg *config.Config, s3 *storage.Client, ctx context.Context) {
func recoverStuck(app core.App, cfg *config.Config, s3 *storage.Client, ctx context.Context) {
stuck, err := app.FindRecordsByFilter("documents", "file_key ~ {:p}", "", 100, 0, dbx.Params{"p": "inbox/"})
if err != nil {
app.Logger().Error("recovery: query failed", "error", err)
@@ -213,7 +213,11 @@ func extractPages(data []byte) ([]string, error) {
if p.V.IsNull() {
return nil, fmt.Errorf("page %d is null", i)
}
pages = append(pages, p.GetPlainText(nil))
text, err := p.GetPlainText(nil)
if err != nil {
return nil, fmt.Errorf("page %d: %w", i, err)
}
pages = append(pages, text)
}
if len(pages) == 0 {
return nil, fmt.Errorf("no page extracted")
archivist/internal/storage/s3.go
+1 -1
View File
@@ -97,7 +97,7 @@ func (c *Client) Exists(ctx context.Context, key string) bool {
func (c *Client) Copy(ctx context.Context, srcKey, dstKey string) error {
_, err := c.mc.CopyObject(ctx,
minio.CopyDestOptions{Bucket: c.Bucket, Object: dstKey},
minio.CopySourceOptions{Bucket: c.Bucket, Object: srcKey})
minio.CopySrcOptions{Bucket: c.Bucket, Object: srcKey})
return err
}