mirror of
https://github.com/napnap75/multiarch-docker-images.git
synced 2026-11-13 09:31:46 +01:00
Corrections after first run
This commit is contained in:
@@ -0,0 +1,5 @@
|
|||||||
|
.vscode/
|
||||||
|
pb_data/
|
||||||
|
tests/
|
||||||
|
__debug_bin*
|
||||||
|
go.sum
|
||||||
@@ -40,3 +40,4 @@ USER 65532:65532
|
|||||||
VOLUME /app/pb_data
|
VOLUME /app/pb_data
|
||||||
EXPOSE 8090
|
EXPOSE 8090
|
||||||
ENTRYPOINT ["/archivist"]
|
ENTRYPOINT ["/archivist"]
|
||||||
|
CMD ["serve"]
|
||||||
|
|||||||
+3
-1
@@ -35,7 +35,7 @@ docker run -d --name archivist \
|
|||||||
-e ARCHIVIST_S3_USE_SSL="true" \
|
-e ARCHIVIST_S3_USE_SSL="true" \
|
||||||
-e ARCHIVIST_ADMIN_EMAIL="me@mydomain.fr" \
|
-e ARCHIVIST_ADMIN_EMAIL="me@mydomain.fr" \
|
||||||
-e ARCHIVIST_ADMIN_PASSWORD="..." \
|
-e ARCHIVIST_ADMIN_PASSWORD="..." \
|
||||||
napnap75/archivist:latest
|
napnap75/archivist:latest serve --config /app/config.jsonc
|
||||||
```
|
```
|
||||||
|
|
||||||
The dashboard is available at `http://<server>:8090/`, the PocketBase admin at `http://<server>:8090/_/`.
|
The dashboard is available at `http://<server>:8090/`, the PocketBase admin at `http://<server>:8090/_/`.
|
||||||
@@ -72,6 +72,8 @@ The functional configuration is reloaded only when the container restarts.
|
|||||||
| `ARCHIVIST_ADMIN_PASSWORD` | yes | Admin account password |
|
| `ARCHIVIST_ADMIN_PASSWORD` | yes | Admin account password |
|
||||||
| `ARCHIVIST_HTTP_ADDR` | no | Listen address, default `0.0.0.0:8090` |
|
| `ARCHIVIST_HTTP_ADDR` | no | Listen address, default `0.0.0.0:8090` |
|
||||||
|
|
||||||
|
The PocketBase command is `serve` (the Docker image uses it by default). To load a different functional config file, pass `--config /path/to/config.jsonc` after `serve`. This overrides `ARCHIVIST_CONFIG`, which defaults to `config.jsonc`.
|
||||||
|
|
||||||
The `pb_data` volume holds the PocketBase SQLite database (documents, history, full-text search index): it must persist across restarts.
|
The `pb_data` volume holds the PocketBase SQLite database (documents, history, full-text search index): it must persist across restarts.
|
||||||
|
|
||||||
### The config.jsonc file
|
### The config.jsonc file
|
||||||
|
|||||||
+50
-6
@@ -1,13 +1,57 @@
|
|||||||
module github.com/napnap75/archivist
|
module github.com/napnap75/archivist
|
||||||
|
|
||||||
go 1.27
|
go 1.27.3
|
||||||
|
|
||||||
// Run `go mod tidy` to resolve the exact compatible versions.
|
|
||||||
require (
|
require (
|
||||||
ganigeorgiev.com/pocketbase/dbx v1.13.0
|
ganigeorgiev.com/pocketbase/dbx v1.13.0
|
||||||
ganigeorgiev.com/pocketbase/pocketbase v0.41.1
|
ganigeorgiev.com/pocketbase/pocketbase v0.41.1
|
||||||
github.com/expr-lang/expr v1.17.6
|
github.com/expr-lang/expr v1.17.8
|
||||||
github.com/ledongthuc/pdf v0.0.0-20240201144150-fd9d85c880aa
|
github.com/ledongthuc/pdf v0.0.0-20261020125853-5e46805f718e
|
||||||
github.com/minio/minio-go/v7 v7.0.88
|
github.com/minio/minio-go/v7 v7.4.1
|
||||||
github.com/tailscale/hujson v0.4.0
|
github.com/spf13/pflag v1.0.10
|
||||||
|
github.com/tailscale/hujson v0.0.0-20260727124030-b80ff77dac4f
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
ganigeorgiev.com/fexpr v0.7.0 // indirect
|
||||||
|
ganigeorgiev.com/pocketbase/ozzo-validation/v4 v4.4.1 // indirect
|
||||||
|
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
||||||
|
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||||
|
github.com/disintegration/imaging v1.6.2 // indirect
|
||||||
|
github.com/domodwyer/mailyak/v3 v3.6.2 // indirect
|
||||||
|
github.com/dustin/go-humanize v1.1.0 // indirect
|
||||||
|
github.com/fatih/color v1.19.0 // indirect
|
||||||
|
github.com/fsnotify/fsnotify v1.10.1 // indirect
|
||||||
|
github.com/gabriel-vasile/mimetype v1.4.15 // indirect
|
||||||
|
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
|
||||||
|
github.com/google/uuid v1.6.0 // indirect
|
||||||
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||||
|
github.com/klauspost/compress v1.19.2 // indirect
|
||||||
|
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
|
||||||
|
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||||
|
github.com/mattn/go-colorable v0.1.16 // indirect
|
||||||
|
github.com/mattn/go-isatty v0.0.24 // indirect
|
||||||
|
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||||
|
github.com/minio/md5-simd v1.1.2 // indirect
|
||||||
|
github.com/ncruces/go-strftime v1.1.0 // indirect
|
||||||
|
github.com/philhofer/fwd v1.2.0 // indirect
|
||||||
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||||
|
github.com/rs/xid v1.6.0 // indirect
|
||||||
|
github.com/spf13/cast v1.10.0 // indirect
|
||||||
|
github.com/spf13/cobra v1.10.2 // indirect
|
||||||
|
github.com/tinylib/msgp v1.6.4 // indirect
|
||||||
|
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||||
|
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||||
|
golang.org/x/crypto v0.58.0 // indirect
|
||||||
|
golang.org/x/image v0.47.0 // indirect
|
||||||
|
golang.org/x/net v0.62.0 // indirect
|
||||||
|
golang.org/x/oauth2 v0.37.0 // indirect
|
||||||
|
golang.org/x/sync v0.24.0 // indirect
|
||||||
|
golang.org/x/sys v0.49.0 // indirect
|
||||||
|
golang.org/x/text v0.43.0 // indirect
|
||||||
|
gopkg.in/ini.v1 v1.67.3 // indirect
|
||||||
|
modernc.org/libc v1.77.1 // indirect
|
||||||
|
modernc.org/mathutil v1.7.1 // indirect
|
||||||
|
modernc.org/memory v1.12.1 // indirect
|
||||||
|
modernc.org/sqlite v1.60.1 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/expr-lang/expr"
|
"github.com/expr-lang/expr"
|
||||||
|
"github.com/expr-lang/expr/vm"
|
||||||
"github.com/tailscale/hujson"
|
"github.com/tailscale/hujson"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -88,6 +89,7 @@ func Load(path string) (*Config, error) {
|
|||||||
c.Fields[f.Name] = f
|
c.Fields[f.Name] = f
|
||||||
c.FieldList = append(c.FieldList, f)
|
c.FieldList = append(c.FieldList, f)
|
||||||
}
|
}
|
||||||
|
SetKnownFields(c.FieldList)
|
||||||
|
|
||||||
// --- rules --------------------------------------------------------------
|
// --- rules --------------------------------------------------------------
|
||||||
if err := validateRules(raw.Rules, c.Fields); err != nil {
|
if err := validateRules(raw.Rules, c.Fields); err != nil {
|
||||||
@@ -124,7 +126,7 @@ func checkRuleTemplates(rules []*Rule, c *Config) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, child := range r.Rules {
|
for _, child := range r.Rules {
|
||||||
if err := checkRuleTemplates(child, c); err != nil {
|
if err := checkRuleTemplates([]*Rule{child}, c); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -166,8 +168,29 @@ func validateRules(rules []*Rule, fields map[string]Field) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// CompileExpr compiles one expression against the typed document context.
|
// CompileExpr compiles one expression against the typed document context.
|
||||||
func CompileExpr(src string) (expr.Node, error) {
|
func CompileExpr(src string) (*vm.Program, error) {
|
||||||
return expr.Compile(src, expr.Env(ExprEnv(nil, nil)))
|
env := ExprEnv(nil, nil)
|
||||||
|
for _, name := range knownFields {
|
||||||
|
switch knownFieldTypes[name] {
|
||||||
|
case "number":
|
||||||
|
env[name] = float64(0)
|
||||||
|
case "date":
|
||||||
|
env[name] = time.Time{}
|
||||||
|
case "boolean":
|
||||||
|
env[name] = false
|
||||||
|
default:
|
||||||
|
env[name] = ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
opts := append([]expr.Option{expr.Env(env)}, exprOptions()...)
|
||||||
|
return expr.Compile(src, opts...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func exprOptions() []expr.Option {
|
||||||
|
return []expr.Option{
|
||||||
|
expr.Function("firstMatch", FirstMatch),
|
||||||
|
expr.Function("period", Period),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExprEnv builds the evaluation environment: pages + declared fields.
|
// ExprEnv builds the evaluation environment: pages + declared fields.
|
||||||
@@ -186,11 +209,14 @@ func ExprEnv(pages []string, fields map[string]any) map[string]any {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var knownFields []string
|
var knownFields []string
|
||||||
|
var knownFieldTypes map[string]string
|
||||||
|
|
||||||
func SetKnownFields(fields []Field) {
|
func SetKnownFields(fields []Field) {
|
||||||
knownFields = nil
|
knownFields = nil
|
||||||
|
knownFieldTypes = make(map[string]string, len(fields))
|
||||||
for _, f := range fields {
|
for _, f := range fields {
|
||||||
knownFields = append(knownFields, f.Name)
|
knownFields = append(knownFields, f.Name)
|
||||||
|
knownFieldTypes[f.Name] = f.Type
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -230,7 +256,12 @@ func (c *Config) Classify(pages []string) (map[string]any, []string) {
|
|||||||
walk = func(rules []*Rule) {
|
walk = func(rules []*Rule) {
|
||||||
for _, r := range rules {
|
for _, r := range rules {
|
||||||
env := ExprEnv(pages, fields)
|
env := ExprEnv(pages, fields)
|
||||||
res, err := expr.Eval(r.When, env)
|
program, err := CompileExpr(r.When)
|
||||||
|
if err != nil {
|
||||||
|
errs = append(errs, fmt.Sprintf("when: %v", err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
res, err := expr.Run(program, env)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errs = append(errs, fmt.Sprintf("when: %v", err))
|
errs = append(errs, fmt.Sprintf("when: %v", err))
|
||||||
continue
|
continue
|
||||||
@@ -241,8 +272,12 @@ func (c *Config) Classify(pages []string) (map[string]any, []string) {
|
|||||||
for _, s := range r.Set {
|
for _, s := range r.Set {
|
||||||
var v any
|
var v any
|
||||||
if s.Eval != "" {
|
if s.Eval != "" {
|
||||||
var err error
|
program, err := CompileExpr(s.Eval)
|
||||||
v, err = expr.Eval(s.Eval, env)
|
if err != nil {
|
||||||
|
errs = append(errs, fmt.Sprintf("eval %s: %v", s.Field, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
v, err = expr.Run(program, env)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errs = append(errs, fmt.Sprintf("eval %s: %v", s.Field, err))
|
errs = append(errs, fmt.Sprintf("eval %s: %v", s.Field, err))
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -0,0 +1,26 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDefaultConfigExpressionHelpers(t *testing.T) {
|
||||||
|
cfg, err := Load("../../config.jsonc")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load default config: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fields, rendered, errs := cfg.Evaluate([]string{
|
||||||
|
"Relevé de compte\nBRED\n12345678901\nDate : 15/01/2026",
|
||||||
|
})
|
||||||
|
if len(errs) != 0 {
|
||||||
|
t.Fatalf("evaluate default config: %v", errs)
|
||||||
|
}
|
||||||
|
if got, ok := fields["emissionDate"].(time.Time); !ok || got.Format("02/01/2006") != "15/01/2026" {
|
||||||
|
t.Fatalf("emissionDate = %v, want 15/01/2026", fields["emissionDate"])
|
||||||
|
}
|
||||||
|
if rendered == "" {
|
||||||
|
t.Fatal("default config rendered an empty file name")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -8,13 +8,14 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/expr-lang/expr"
|
"github.com/expr-lang/expr"
|
||||||
|
"github.com/expr-lang/expr/vm"
|
||||||
)
|
)
|
||||||
|
|
||||||
var placeholderRe = regexp.MustCompile(`\{([^{}]+)\}`)
|
var placeholderRe = regexp.MustCompile(`\{([^{}]+)\}`)
|
||||||
|
|
||||||
type tplPart struct {
|
type tplPart struct {
|
||||||
literal string
|
literal string
|
||||||
program expr.Node
|
program *vm.Program
|
||||||
hasExpr bool
|
hasExpr bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -5,28 +5,28 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
ganigeorgiev.com/pocketbase/pocketbase/core"
|
"ganigeorgiev.com/pocketbase/pocketbase/core"
|
||||||
"github.com/napnap75/archivist/internal/config"
|
"github.com/napnap75/archivist/internal/config"
|
||||||
"github.com/napnap75/archivist/internal/storage"
|
"github.com/napnap75/archivist/internal/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
// BindHooks registers the request hooks (US-04, US-22, US-23).
|
// BindHooks registers the request hooks (US-04, US-22, US-23).
|
||||||
func BindHooks(app *core.App, cfg *config.Config, s3 *storage.Client) {
|
func BindHooks(app core.App, cfg *config.Config, s3 *storage.Client) {
|
||||||
// US-04: document_events is append-only for everyone, superusers included.
|
// US-04: document_events is append-only for everyone, superusers included.
|
||||||
app.OnRecordUpdateRequest(eventsName).PreBindFunc(func(e *core.RecordRequestEvent) error {
|
app.OnRecordUpdateRequest(eventsName).BindFunc(func(e *core.RecordRequestEvent) error {
|
||||||
return errAppendOnly
|
return errAppendOnly
|
||||||
})
|
})
|
||||||
app.OnRecordDeleteRequest(eventsName).PreBindFunc(func(e *core.RecordRequestEvent) error {
|
app.OnRecordDeleteRequest(eventsName).BindFunc(func(e *core.RecordRequestEvent) error {
|
||||||
return errAppendOnly
|
return errAppendOnly
|
||||||
})
|
})
|
||||||
|
|
||||||
// US-22 / US-23: corrections and validations through the standard UI.
|
// US-22 / US-23: corrections and validations through the standard UI.
|
||||||
app.OnRecordUpdateRequest("documents").PreBindFunc(func(e *core.RecordRequestEvent) error {
|
app.OnRecordUpdateRequest("documents").BindFunc(func(e *core.RecordRequestEvent) error {
|
||||||
return handleDocumentUpdate(app, cfg, s3, e)
|
return handleDocumentUpdate(app, cfg, s3, e)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func handleDocumentUpdate(app *core.App, cfg *config.Config, s3 *storage.Client, e *core.RecordRequestEvent) error {
|
func handleDocumentUpdate(app core.App, cfg *config.Config, s3 *storage.Client, e *core.RecordRequestEvent) error {
|
||||||
newRec := e.Record
|
newRec := e.Record
|
||||||
|
|
||||||
// The original state, before the submitted data is persisted.
|
// The original state, before the submitted data is persisted.
|
||||||
@@ -158,7 +158,7 @@ type Payload struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// AppendEvent appends one event to document_events.
|
// AppendEvent appends one event to document_events.
|
||||||
func AppendEvent(app *core.App, documentId, eventType string, before, after Snapshot, errs []string) error {
|
func AppendEvent(app core.App, documentId, eventType string, before, after Snapshot, errs []string) error {
|
||||||
coll, err := app.FindCollectionByNameOrId(eventsName)
|
coll, err := app.FindCollectionByNameOrId(eventsName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -184,7 +184,7 @@ func AppendEvent(app *core.App, documentId, eventType string, before, after Snap
|
|||||||
}
|
}
|
||||||
|
|
||||||
// AppendImportedEvent builds the imported event of US-16.
|
// AppendImportedEvent builds the imported event of US-16.
|
||||||
func AppendImportedEvent(app *core.App, documentId, fromKey, toKey string, valid bool, fields map[string]any, errs []string) error {
|
func AppendImportedEvent(app core.App, documentId, fromKey, toKey string, valid bool, fields map[string]any, errs []string) error {
|
||||||
before := Snapshot{"file_key": fromKey}
|
before := Snapshot{"file_key": fromKey}
|
||||||
after := Snapshot{"file_key": toKey}
|
after := Snapshot{"file_key": toKey}
|
||||||
if valid {
|
if valid {
|
||||||
|
|||||||
@@ -6,14 +6,14 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
ganigeorgiev.com/pocketbase/pocketbase/core"
|
"ganigeorgiev.com/pocketbase/pocketbase/core"
|
||||||
"github.com/napnap75/archivist/internal/config"
|
"github.com/napnap75/archivist/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
const eventsName = "document_events"
|
const eventsName = "document_events"
|
||||||
|
|
||||||
// Setup ensures the collections and the superuser account (US-03, US-04, US-01).
|
// Setup ensures the collections and the superuser account (US-03, US-04, US-01).
|
||||||
func Setup(app *core.App, adminEmail, adminPassword string, cfg *config.Config) error {
|
func Setup(app core.App, adminEmail, adminPassword string, cfg *config.Config) error {
|
||||||
if err := ensureDocuments(app, cfg); err != nil {
|
if err := ensureDocuments(app, cfg); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -23,11 +23,10 @@ func Setup(app *core.App, adminEmail, adminPassword string, cfg *config.Config)
|
|||||||
if err := ensureSuperuser(app, adminEmail, adminPassword); err != nil {
|
if err := ensureSuperuser(app, adminEmail, adminPassword); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
config.SetKnownFields(cfg.FieldList)
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func ensureDocuments(app *core.App, cfg *config.Config) error {
|
func ensureDocuments(app core.App, cfg *config.Config) error {
|
||||||
coll, err := app.FindCollectionByNameOrId("documents")
|
coll, err := app.FindCollectionByNameOrId("documents")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// First start: create the collection with its fixed columns (US-03).
|
// First start: create the collection with its fixed columns (US-03).
|
||||||
@@ -59,9 +58,9 @@ func ensureDocuments(app *core.App, cfg *config.Config) error {
|
|||||||
case "boolean":
|
case "boolean":
|
||||||
newField = &core.BoolField{Name: f.Name}
|
newField = &core.BoolField{Name: f.Name}
|
||||||
case "enum":
|
case "enum":
|
||||||
// The vocabulary is an application notion (dashboard dropdown with
|
// PocketBase select fields require a fixed non-empty vocabulary;
|
||||||
// free entry), not a schema constraint: empty allowed values.
|
// config enums are free-form, so store them as text.
|
||||||
newField = &core.SelectField{Name: f.Name, MaxSelect: 1}
|
newField = &core.TextField{Name: f.Name}
|
||||||
}
|
}
|
||||||
if existing := coll.Fields.GetByName(f.Name); existing != nil {
|
if existing := coll.Fields.GetByName(f.Name); existing != nil {
|
||||||
if existing.Type() != newField.Type() {
|
if existing.Type() != newField.Type() {
|
||||||
@@ -79,9 +78,9 @@ func ensureDocuments(app *core.App, cfg *config.Config) error {
|
|||||||
if f.GetSystem() {
|
if f.GetSystem() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if _, declared := cfg.Fields[f.Name]; !declared && !isFixed(f.Name) {
|
if _, declared := cfg.Fields[f.GetName()]; !declared && !isFixed(f.GetName()) {
|
||||||
app.Logger().Warn("documents: column removed", "field", f.Name, "reason", "removed from config")
|
app.Logger().Warn("documents: column removed", "field", f.GetName(), "reason", "removed from config")
|
||||||
coll.Fields.RemoveByName(f.Name)
|
coll.Fields.RemoveByName(f.GetName())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -104,7 +103,7 @@ func isFixed(name string) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func ensureEvents(app *core.App) error {
|
func ensureEvents(app core.App) error {
|
||||||
if _, err := app.FindCollectionByNameOrId(eventsName); err == nil {
|
if _, err := app.FindCollectionByNameOrId(eventsName); err == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -128,7 +127,7 @@ func ensureEvents(app *core.App) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func ensureSuperuser(app *core.App, email, password string) error {
|
func ensureSuperuser(app core.App, email, password string) error {
|
||||||
superusers, err := app.FindCollectionByNameOrId(core.CollectionNameSuperusers)
|
superusers, err := app.FindCollectionByNameOrId(core.CollectionNameSuperusers)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -0,0 +1,42 @@
|
|||||||
|
package pbapp
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"ganigeorgiev.com/pocketbase/pocketbase"
|
||||||
|
"github.com/napnap75/archivist/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestEnsureDocumentsUsesTextForFreeFormEnums(t *testing.T) {
|
||||||
|
app := pocketbase.NewWithConfig(pocketbase.Config{DefaultDataDir: t.TempDir()})
|
||||||
|
if err := app.Bootstrap(); err != nil {
|
||||||
|
t.Fatalf("bootstrap PocketBase: %v", err)
|
||||||
|
}
|
||||||
|
defer app.ClearBootstrap()
|
||||||
|
|
||||||
|
fields := []config.Field{
|
||||||
|
{Name: "documentType", Type: "enum"},
|
||||||
|
{Name: "bankAccount", Type: "enum"},
|
||||||
|
}
|
||||||
|
cfg := &config.Config{
|
||||||
|
Fields: map[string]config.Field{
|
||||||
|
"documentType": fields[0],
|
||||||
|
"bankAccount": fields[1],
|
||||||
|
},
|
||||||
|
FieldList: fields,
|
||||||
|
}
|
||||||
|
if err := ensureDocuments(app, cfg); err != nil {
|
||||||
|
t.Fatalf("ensure documents: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
coll, err := app.FindCollectionByNameOrId("documents")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("find documents collection: %v", err)
|
||||||
|
}
|
||||||
|
for _, name := range []string{"documentType", "bankAccount"} {
|
||||||
|
field := coll.Fields.GetByName(name)
|
||||||
|
if field == nil || field.Type() != "text" {
|
||||||
|
t.Errorf("field %q = %v, want text field", name, field)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -8,8 +8,8 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
ganigeorgiev.com/pocketbase/dbx"
|
"ganigeorgiev.com/pocketbase/dbx"
|
||||||
ganigeorgiev.com/pocketbase/pocketbase/core"
|
"ganigeorgiev.com/pocketbase/pocketbase/core"
|
||||||
"github.com/ledongthuc/pdf"
|
"github.com/ledongthuc/pdf"
|
||||||
"github.com/napnap75/archivist/internal/config"
|
"github.com/napnap75/archivist/internal/config"
|
||||||
"github.com/napnap75/archivist/internal/pbapp"
|
"github.com/napnap75/archivist/internal/pbapp"
|
||||||
@@ -24,7 +24,7 @@ type pollState struct {
|
|||||||
|
|
||||||
var stable = map[string]pollState{}
|
var stable = map[string]pollState{}
|
||||||
|
|
||||||
func RunPoll(app *core.App, cfg *config.Config, s3 *storage.Client) {
|
func RunPoll(app core.App, cfg *config.Config, s3 *storage.Client) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
// US-11: recover interrupted filings first.
|
// US-11: recover interrupted filings first.
|
||||||
@@ -54,7 +54,7 @@ func RunPoll(app *core.App, cfg *config.Config, s3 *storage.Client) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func processOne(app *core.App, cfg *config.Config, s3 *storage.Client, ctx context.Context, key string) error {
|
func processOne(app core.App, cfg *config.Config, s3 *storage.Client, ctx context.Context, key string) error {
|
||||||
// US-12: download in memory, sha256 on the fly.
|
// US-12: download in memory, sha256 on the fly.
|
||||||
data, hash, err := s3.Get(ctx, key)
|
data, hash, err := s3.Get(ctx, key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -154,7 +154,7 @@ func fileDocument(s3 *storage.Client, ctx context.Context, record *core.Record,
|
|||||||
|
|
||||||
// recoverStuck completes interrupted filings: records whose file_key is still
|
// recoverStuck completes interrupted filings: records whose file_key is still
|
||||||
// under inbox/ (US-11).
|
// under inbox/ (US-11).
|
||||||
func recoverStuck(app *core.App, cfg *config.Config, s3 *storage.Client, ctx context.Context) {
|
func recoverStuck(app core.App, cfg *config.Config, s3 *storage.Client, ctx context.Context) {
|
||||||
stuck, err := app.FindRecordsByFilter("documents", "file_key ~ {:p}", "", 100, 0, dbx.Params{"p": "inbox/"})
|
stuck, err := app.FindRecordsByFilter("documents", "file_key ~ {:p}", "", 100, 0, dbx.Params{"p": "inbox/"})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
app.Logger().Error("recovery: query failed", "error", err)
|
app.Logger().Error("recovery: query failed", "error", err)
|
||||||
@@ -213,7 +213,11 @@ func extractPages(data []byte) ([]string, error) {
|
|||||||
if p.V.IsNull() {
|
if p.V.IsNull() {
|
||||||
return nil, fmt.Errorf("page %d is null", i)
|
return nil, fmt.Errorf("page %d is null", i)
|
||||||
}
|
}
|
||||||
pages = append(pages, p.GetPlainText(nil))
|
text, err := p.GetPlainText(nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("page %d: %w", i, err)
|
||||||
|
}
|
||||||
|
pages = append(pages, text)
|
||||||
}
|
}
|
||||||
if len(pages) == 0 {
|
if len(pages) == 0 {
|
||||||
return nil, fmt.Errorf("no page extracted")
|
return nil, fmt.Errorf("no page extracted")
|
||||||
|
|||||||
@@ -97,7 +97,7 @@ func (c *Client) Exists(ctx context.Context, key string) bool {
|
|||||||
func (c *Client) Copy(ctx context.Context, srcKey, dstKey string) error {
|
func (c *Client) Copy(ctx context.Context, srcKey, dstKey string) error {
|
||||||
_, err := c.mc.CopyObject(ctx,
|
_, err := c.mc.CopyObject(ctx,
|
||||||
minio.CopyDestOptions{Bucket: c.Bucket, Object: dstKey},
|
minio.CopyDestOptions{Bucket: c.Bucket, Object: dstKey},
|
||||||
minio.CopySourceOptions{Bucket: c.Bucket, Object: srcKey})
|
minio.CopySrcOptions{Bucket: c.Bucket, Object: srcKey})
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+24
-4
@@ -4,12 +4,13 @@ import (
|
|||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
ganigeorgiev.com/pocketbase/pocketbase"
|
"ganigeorgiev.com/pocketbase/pocketbase"
|
||||||
ganigeorgiev.com/pocketbase/pocketbase/core"
|
"ganigeorgiev.com/pocketbase/pocketbase/core"
|
||||||
"github.com/napnap75/archivist/internal/config"
|
"github.com/napnap75/archivist/internal/config"
|
||||||
"github.com/napnap75/archivist/internal/pbapp"
|
"github.com/napnap75/archivist/internal/pbapp"
|
||||||
"github.com/napnap75/archivist/internal/pipeline"
|
"github.com/napnap75/archivist/internal/pipeline"
|
||||||
"github.com/napnap75/archivist/internal/storage"
|
"github.com/napnap75/archivist/internal/storage"
|
||||||
|
"github.com/spf13/pflag"
|
||||||
)
|
)
|
||||||
|
|
||||||
// EnvConfig is the technical configuration, from environment variables (US-01).
|
// EnvConfig is the technical configuration, from environment variables (US-01).
|
||||||
@@ -64,11 +65,27 @@ func LoadEnvConfig() (*EnvConfig, error) {
|
|||||||
return c, nil
|
return c, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func configPathFromArgs(args []string, defaultPath string) (string, error) {
|
||||||
|
var configPath string
|
||||||
|
flags := pflag.NewFlagSet("archivist", pflag.ContinueOnError)
|
||||||
|
flags.SetInterspersed(true)
|
||||||
|
flags.ParseErrorsWhitelist.UnknownFlags = true
|
||||||
|
flags.StringVar(&configPath, "config", defaultPath, "Path to the functional JSONC configuration")
|
||||||
|
if err := flags.Parse(args); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return configPath, nil
|
||||||
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
envCfg, err := LoadEnvConfig()
|
envCfg, err := LoadEnvConfig()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
|
envCfg.ConfigPath, err = configPathFromArgs(os.Args[1:], envCfg.ConfigPath)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("invalid command line: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
// US-02: load and validate the functional configuration. Full rejection,
|
// US-02: load and validate the functional configuration. Full rejection,
|
||||||
// the process does not continue with a partial configuration (US-05).
|
// the process does not continue with a partial configuration (US-05).
|
||||||
@@ -86,6 +103,7 @@ func main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
app := pocketbase.New()
|
app := pocketbase.New()
|
||||||
|
app.RootCmd.PersistentFlags().StringVar(&envCfg.ConfigPath, "config", envCfg.ConfigPath, "Path to the functional JSONC configuration")
|
||||||
// The PocketBase logger and database are available only after bootstrap;
|
// The PocketBase logger and database are available only after bootstrap;
|
||||||
// everything logged before this point goes to stderr (arch.md section 9).
|
// everything logged before this point goes to stderr (arch.md section 9).
|
||||||
if err := app.Bootstrap(); err != nil {
|
if err := app.Bootstrap(); err != nil {
|
||||||
@@ -119,9 +137,11 @@ func main() {
|
|||||||
return e.Next()
|
return e.Next()
|
||||||
})
|
})
|
||||||
// US-10..US-17: the poller, on the configured cron.
|
// US-10..US-17: the poller, on the configured cron.
|
||||||
app.CronAdd(envCfg.PollCron, "archivist-poller", func() {
|
if err := app.Cron().Add("archivist-poller", envCfg.PollCron, func() {
|
||||||
pipeline.RunPoll(app, cfg, s3)
|
pipeline.RunPoll(app, cfg, s3)
|
||||||
})
|
}); err != nil {
|
||||||
|
log.Fatalf("register archivist poller: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
app.RootCmd.PersistentFlags().Set("http", envCfg.HTTPAddr)
|
app.RootCmd.PersistentFlags().Set("http", envCfg.HTTPAddr)
|
||||||
if err := app.Start(); err != nil {
|
if err := app.Start(); err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestConfigPathFromArgs(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
defaultPath string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "default path",
|
||||||
|
args: []string{"serve"},
|
||||||
|
defaultPath: "config.jsonc",
|
||||||
|
want: "config.jsonc",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "config after serve",
|
||||||
|
args: []string{"serve", "--config", "/etc/archivist/custom.jsonc"},
|
||||||
|
defaultPath: "config.jsonc",
|
||||||
|
want: "/etc/archivist/custom.jsonc",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "config among PocketBase flags",
|
||||||
|
args: []string{"serve", "--http=127.0.0.1:8090", "--config=/etc/archivist/custom.jsonc"},
|
||||||
|
defaultPath: "config.jsonc",
|
||||||
|
want: "/etc/archivist/custom.jsonc",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got, err := configPathFromArgs(tt.args, tt.defaultPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("configPathFromArgs() error = %v", err)
|
||||||
|
}
|
||||||
|
if got != tt.want {
|
||||||
|
t.Errorf("configPathFromArgs() = %q, want %q", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user