mirror of
https://github.com/napnap75/multiarch-docker-images.git
synced 2026-11-16 19:01:48 +01:00
Archivist : page Outils + outil « Recherche des fichiers perdus » (US-40, US-41) (#16)
* Add tools page with find-lost-files consistency tool (US-40, US-41) Co-authored-by: napnap75 <napnap75@users.noreply.github.com> * tools page: hide the job panel when there is nothing to show, guard against undefined rendering Co-authored-by: napnap75 <napnap75@users.noreply.github.com> * inbox: leave non-PDF objects in place, never imported (US-12) Co-authored-by: napnap75 <napnap75@users.noreply.github.com> --------- Co-authored-by: Vibe Nuage Agent <vibe@mistral.ai> Co-authored-by: napnap75 <napnap75@users.noreply.github.com>
This commit is contained in:
co-authored by
napnap75
Vibe Nuage Agent
parent
85784ef578
commit
1224ff916d
@@ -3,6 +3,7 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -24,6 +25,18 @@ type pollState struct {
|
||||
|
||||
var stable = map[string]pollState{}
|
||||
|
||||
// nonPDF remembers the objects already found not to be PDFs (key -> ETag at
|
||||
// the check): they are left in place, so the pass skips them without
|
||||
// re-downloading until their content changes.
|
||||
var nonPDF = map[string]string{}
|
||||
|
||||
var pdfMagic = []byte("%PDF-")
|
||||
|
||||
// isPDF reports whether the content carries the PDF header.
|
||||
func isPDF(data []byte) bool {
|
||||
return bytes.HasPrefix(data, pdfMagic)
|
||||
}
|
||||
|
||||
func RunPoll(app core.App, cfg *config.Config, s3 *storage.Client) {
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -48,19 +61,33 @@ func RunPoll(app core.App, cfg *config.Config, s3 *storage.Client) {
|
||||
continue // first sighting, or still uploading: wait for stability
|
||||
}
|
||||
delete(stable, o.Key)
|
||||
if err := processOne(app, cfg, s3, ctx, o.Key); err != nil {
|
||||
if err := processOne(app, cfg, s3, ctx, o.Key, o.ETag); err != nil {
|
||||
app.Logger().Error("poll: processing failed", "key", o.Key, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func processOne(app core.App, cfg *config.Config, s3 *storage.Client, ctx context.Context, key string) error {
|
||||
func processOne(app core.App, cfg *config.Config, s3 *storage.Client, ctx context.Context, key, etag string) error {
|
||||
// Already found not to be a PDF and unchanged since: left in place,
|
||||
// nothing to do.
|
||||
if nonPDF[key] == etag {
|
||||
return nil
|
||||
}
|
||||
|
||||
// US-12: download in memory, sha256 on the fly.
|
||||
data, hash, err := s3.Get(ctx, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("download: %w", err)
|
||||
}
|
||||
|
||||
// Not a PDF: the object is left as-is in inbox/, never imported — no
|
||||
// record, no event; the skip is logged once per unchanged content.
|
||||
if !isPDF(data) {
|
||||
nonPDF[key] = etag
|
||||
app.Logger().Warn("inbox: object is not a PDF, left in place", "key", key)
|
||||
return nil
|
||||
}
|
||||
|
||||
// US-12: duplicate rejection.
|
||||
dup, err := app.FindFirstRecordByFilter("documents", "sha256 = {:h}", dbx.Params{"h": hash})
|
||||
if err == nil && dup != nil {
|
||||
|
||||
Reference in New Issue
Block a user