mirror of
https://github.com/napnap75/multiarch-docker-images.git
synced 2026-11-11 00:22:12 +01:00
feat(events): store before/after as diffs, not full snapshots
AppendEvent now reduces the before/after snapshots to the changed keys: identical values on both sides are omitted, a key on one side only is an addition or a deletion. The imported, corrected and validated payloads all follow the same rule (an import is a diff from nothing, a validation keeps carrying the status alone). arch.md section 4 updated accordingly.
This commit is contained in:
+2
-2
@@ -56,13 +56,13 @@ The append-only `document_events` collection is the single source of truth for a
|
||||
### `document_events`
|
||||
Append-only history: `document`, `event_type` (`imported`, `corrected`, `validated`), `payload` as JSON, `timestamp`.
|
||||
|
||||
The payload is self-contained: each event carries its own type, its own timestamp and the document state before and after, so a document's full history is reconstructed from this collection alone. All dates are RFC 3339; all S3 keys are complete, prefix included.
|
||||
The payload is self-contained: each event carries its own type, its own timestamp and the document state before and after, so a document's full history is reconstructed from this collection alone; the snapshots are stored as diffs — each `before`/`after` carries only the fields the event changes. All dates are RFC 3339; all S3 keys are complete, prefix included.
|
||||
|
||||
Common payload keys:
|
||||
|
||||
- `event_type`: `imported` | `corrected` | `validated`
|
||||
- `timestamp`: RFC 3339 date of the event
|
||||
- `before` / `after`: snapshots of the document state; `file_key` (the S3 key, unified with the `documents` record field) is always present; `status` appears only when the event changes it — at import the status does not exist before, so `before` carries no `status`; declared extraction fields appear only when they have a recognized value — a missing key means the value is unknown, there is no `null`
|
||||
- `before` / `after`: the event's diff, never full snapshots — a key appears on a side only when its value changes in the event: same value on both sides, it is omitted; absent from `before` and present in `after`, the field was added (every `after` key at import, since nothing existed before); present in `before` and absent from `after`, it was deleted. `file_key` (the S3 key, unified with the `documents` record field) follows the same rule and appears only when the object actually moves; at import the status does not exist before, so `before` carries no `status`. A missing key means the value is unknown, there is no `null`
|
||||
|
||||
Per event type:
|
||||
|
||||
|
||||
@@ -130,11 +130,42 @@ func FieldsFromRecord(cfg *config.Config, record *core.Record) map[string]any {
|
||||
return out
|
||||
}
|
||||
|
||||
// DiffSnapshots reduces two snapshots to the keys whose value differs
|
||||
// between them: a key with the same value on both sides is omitted, a
|
||||
// key present on one side only is kept on that side (added or deleted
|
||||
field). It is the section-4 payload rule: before/after carry only what
|
||||
// the event changes.
|
||||
func DiffSnapshots(before, after Snapshot) (Snapshot, Snapshot) {
|
||||
keys := map[string]bool{}
|
||||
for k := range before {
|
||||
keys[k] = true
|
||||
}
|
||||
for k := range after {
|
||||
keys[k] = true
|
||||
}
|
||||
db, da := Snapshot{}, Snapshot{}
|
||||
for k := range keys {
|
||||
bv, bok := before[k]
|
||||
av, aok := after[k]
|
||||
if bok && aok && fmt.Sprintf("%v", bv) == fmt.Sprintf("%v", av) {
|
||||
continue
|
||||
}
|
||||
if bok {
|
||||
db[k] = bv
|
||||
}
|
||||
if aok {
|
||||
da[k] = av
|
||||
}
|
||||
}
|
||||
return db, da
|
||||
}
|
||||
|
||||
// Snapshot is one before/after state of a document (arch.md §4).
|
||||
type Snapshot map[string]any
|
||||
|
||||
// SnapshotFromRecord builds a snapshot: file_key always, status when it is
|
||||
// set, and the recognized declared fields.
|
||||
// SnapshotFromRecord builds the full snapshot of a record: file_key, status
|
||||
// when set, and the recognized declared fields. AppendEvent reduces pairs of
|
||||
// such snapshots to what actually changes.
|
||||
func SnapshotFromRecord(cfg *config.Config, record *core.Record, withStatus bool) Snapshot {
|
||||
s := Snapshot{"file_key": record.GetString("file_key")}
|
||||
if withStatus {
|
||||
@@ -164,6 +195,9 @@ func AppendEvent(app core.App, documentId, eventType string, before, after Snaps
|
||||
return err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
// The stored payload carries the event's diff only: unchanged keys
|
||||
// are dropped from both sides (arch.md section 4).
|
||||
before, after = DiffSnapshots(before, after)
|
||||
p := Payload{
|
||||
EventType: eventType,
|
||||
Timestamp: now.Format(time.RFC3339),
|
||||
|
||||
Reference in New Issue
Block a user