mirror of
https://github.com/napnap75/multiarch-docker-images.git
synced 2026-11-13 09:31:46 +01:00
feat(events): store before/after as diffs, not full snapshots
AppendEvent now reduces the before/after snapshots to the changed keys: identical values on both sides are omitted, a key on one side only is an addition or a deletion. The imported, corrected and validated payloads all follow the same rule (an import is a diff from nothing, a validation keeps carrying the status alone). arch.md section 4 updated accordingly.
This commit is contained in:
+2
-2
@@ -56,13 +56,13 @@ The append-only `document_events` collection is the single source of truth for a
|
|||||||
### `document_events`
|
### `document_events`
|
||||||
Append-only history: `document`, `event_type` (`imported`, `corrected`, `validated`), `payload` as JSON, `timestamp`.
|
Append-only history: `document`, `event_type` (`imported`, `corrected`, `validated`), `payload` as JSON, `timestamp`.
|
||||||
|
|
||||||
The payload is self-contained: each event carries its own type, its own timestamp and the document state before and after, so a document's full history is reconstructed from this collection alone. All dates are RFC 3339; all S3 keys are complete, prefix included.
|
The payload is self-contained: each event carries its own type, its own timestamp and the document state before and after, so a document's full history is reconstructed from this collection alone; the snapshots are stored as diffs — each `before`/`after` carries only the fields the event changes. All dates are RFC 3339; all S3 keys are complete, prefix included.
|
||||||
|
|
||||||
Common payload keys:
|
Common payload keys:
|
||||||
|
|
||||||
- `event_type`: `imported` | `corrected` | `validated`
|
- `event_type`: `imported` | `corrected` | `validated`
|
||||||
- `timestamp`: RFC 3339 date of the event
|
- `timestamp`: RFC 3339 date of the event
|
||||||
- `before` / `after`: snapshots of the document state; `file_key` (the S3 key, unified with the `documents` record field) is always present; `status` appears only when the event changes it — at import the status does not exist before, so `before` carries no `status`; declared extraction fields appear only when they have a recognized value — a missing key means the value is unknown, there is no `null`
|
- `before` / `after`: the event's diff, never full snapshots — a key appears on a side only when its value changes in the event: same value on both sides, it is omitted; absent from `before` and present in `after`, the field was added (every `after` key at import, since nothing existed before); present in `before` and absent from `after`, it was deleted. `file_key` (the S3 key, unified with the `documents` record field) follows the same rule and appears only when the object actually moves; at import the status does not exist before, so `before` carries no `status`. A missing key means the value is unknown, there is no `null`
|
||||||
|
|
||||||
Per event type:
|
Per event type:
|
||||||
|
|
||||||
|
|||||||
@@ -130,11 +130,42 @@ func FieldsFromRecord(cfg *config.Config, record *core.Record) map[string]any {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DiffSnapshots reduces two snapshots to the keys whose value differs
|
||||||
|
// between them: a key with the same value on both sides is omitted, a
|
||||||
|
// key present on one side only is kept on that side (added or deleted
|
||||||
|
field). It is the section-4 payload rule: before/after carry only what
|
||||||
|
// the event changes.
|
||||||
|
func DiffSnapshots(before, after Snapshot) (Snapshot, Snapshot) {
|
||||||
|
keys := map[string]bool{}
|
||||||
|
for k := range before {
|
||||||
|
keys[k] = true
|
||||||
|
}
|
||||||
|
for k := range after {
|
||||||
|
keys[k] = true
|
||||||
|
}
|
||||||
|
db, da := Snapshot{}, Snapshot{}
|
||||||
|
for k := range keys {
|
||||||
|
bv, bok := before[k]
|
||||||
|
av, aok := after[k]
|
||||||
|
if bok && aok && fmt.Sprintf("%v", bv) == fmt.Sprintf("%v", av) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if bok {
|
||||||
|
db[k] = bv
|
||||||
|
}
|
||||||
|
if aok {
|
||||||
|
da[k] = av
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return db, da
|
||||||
|
}
|
||||||
|
|
||||||
// Snapshot is one before/after state of a document (arch.md §4).
|
// Snapshot is one before/after state of a document (arch.md §4).
|
||||||
type Snapshot map[string]any
|
type Snapshot map[string]any
|
||||||
|
|
||||||
// SnapshotFromRecord builds a snapshot: file_key always, status when it is
|
// SnapshotFromRecord builds the full snapshot of a record: file_key, status
|
||||||
// set, and the recognized declared fields.
|
// when set, and the recognized declared fields. AppendEvent reduces pairs of
|
||||||
|
// such snapshots to what actually changes.
|
||||||
func SnapshotFromRecord(cfg *config.Config, record *core.Record, withStatus bool) Snapshot {
|
func SnapshotFromRecord(cfg *config.Config, record *core.Record, withStatus bool) Snapshot {
|
||||||
s := Snapshot{"file_key": record.GetString("file_key")}
|
s := Snapshot{"file_key": record.GetString("file_key")}
|
||||||
if withStatus {
|
if withStatus {
|
||||||
@@ -164,6 +195,9 @@ func AppendEvent(app core.App, documentId, eventType string, before, after Snaps
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
|
// The stored payload carries the event's diff only: unchanged keys
|
||||||
|
// are dropped from both sides (arch.md section 4).
|
||||||
|
before, after = DiffSnapshots(before, after)
|
||||||
p := Payload{
|
p := Payload{
|
||||||
EventType: eventType,
|
EventType: eventType,
|
||||||
Timestamp: now.Format(time.RFC3339),
|
Timestamp: now.Format(time.RFC3339),
|
||||||
|
|||||||
Reference in New Issue
Block a user