From aab30a17fe31269977fd13cd60ddaaeac87e883a Mon Sep 17 00:00:00 2001 From: Stefan Walcz Date: Sun, 18 Oct 2026 09:45:30 +0200 Subject: [PATCH] feat(pii): optionally block content the detectors cannot inspect The PII middleware scans the text of every message, but images, audio, video and files pass to the backend unchecked. For an egress policy ("nothing sensitive leaves the box") that is a hole: a photographed invoice or an attached PDF reaches a cloud-proxy model without any check. Add pii.block_unscanned_media. While PII is enabled for a model and the option is set, a request that carries such content is rejected with the usual 400 pii_blocked response and an audit event with pattern __unscanned_media__, before any detector runs. Adapters report the kinds of unscanned content through a new optional Adapter.Unscanned: OpenAI chat (non-text content blocks), Anthropic messages (image and document blocks; tool blocks stay allowed) and Ollama chat (images). Off by default, so existing configurations behave as before. Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Stefan Walcz --- core/config/meta/registry.go | 7 ++ core/config/model_config.go | 8 ++ core/services/routing/pii/middleware.go | 64 +++++++++++ .../routing/pii/unscanned_media_test.go | 100 ++++++++++++++++++ core/services/routing/piiadapter/anthropic.go | 19 ++++ core/services/routing/piiadapter/ollama.go | 12 +++ core/services/routing/piiadapter/openai.go | 17 +++ core/services/routing/piiadapter/unscanned.go | 20 ++++ .../routing/piiadapter/unscanned_test.go | 58 ++++++++++ docs/content/operations/middleware.md | 9 ++ 10 files changed, 314 insertions(+) create mode 100644 core/services/routing/pii/unscanned_media_test.go create mode 100644 core/services/routing/piiadapter/unscanned.go create mode 100644 core/services/routing/piiadapter/unscanned_test.go diff --git a/core/config/meta/registry.go b/core/config/meta/registry.go index ea900d79e5ad..f89c0d3f95ea 100644 --- a/core/config/meta/registry.go +++ b/core/config/meta/registry.go @@ -1008,6 +1008,13 @@ func DefaultRegistry() map[string]FieldMetaOverride { Description: "Suffix for reversible redaction tokens. Defaults to ].", Order: 204, }, + "pii.block_unscanned_media": { + Section: "pii", + Label: "Block Unscanned Media", + Description: "Reject requests that carry content the detectors cannot inspect (images, audio, video, files) instead of forwarding it unchecked. Only applies while PII is enabled for this model.", + Component: "toggle", + Order: 205, + }, // --- PII detection policy (on a token_classify detector model) --- "pii_detection.min_score": { diff --git a/core/config/model_config.go b/core/config/model_config.go index a52618fd2351..247b97093085 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -622,11 +622,19 @@ type PIIConfig struct { ReversibleRedactions bool `yaml:"reversible_redactions,omitempty" json:"reversible_redactions,omitempty"` ReversibleTokenPrefix string `yaml:"reversible_token_prefix,omitempty" json:"reversible_token_prefix,omitempty"` ReversibleTokenSuffix string `yaml:"reversible_token_suffix,omitempty" json:"reversible_token_suffix,omitempty"` + + // BlockUnscannedMedia rejects requests that carry content the text + // detectors cannot inspect (images, audio, video, files) instead of + // forwarding it unchecked. Only applies while PII is enabled for the + // model. Meant for egress policies where "not inspectable" must mean + // "not sent", e.g. cloud-proxy models. + BlockUnscannedMedia bool `yaml:"block_unscanned_media,omitempty" json:"block_unscanned_media,omitempty"` } func (c ModelConfig) PIIReversibleRedactions() bool { return c.PII.ReversibleRedactions } func (c ModelConfig) PIIReversibleTokenPrefix() string { return c.PII.ReversibleTokenPrefix } func (c ModelConfig) PIIReversibleTokenSuffix() string { return c.PII.ReversibleTokenSuffix } +func (c ModelConfig) PIIBlockUnscannedMedia() bool { return c.PII.BlockUnscannedMedia } // @Description Detection policy for a token-classification (NER) model // used as a PII detector. Lives on the detector model's own config so the diff --git a/core/services/routing/pii/middleware.go b/core/services/routing/pii/middleware.go index bf77d916d4c0..17aa8ffdc492 100644 --- a/core/services/routing/pii/middleware.go +++ b/core/services/routing/pii/middleware.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "encoding/hex" "net/http" + "strings" "time" "github.com/labstack/echo/v4" @@ -107,8 +108,22 @@ type ScannedText struct { type Adapter struct { Scan func(parsed any) []ScannedText Apply func(parsed any, updates []ScannedText) + // Unscanned lists the kinds of content in the request that Scan does + // not return as text (e.g. "image_url", "input_file"). Optional; used + // only when the model sets pii.block_unscanned_media. + Unscanned func(parsed any) []string } +// unscannedMediaPolicy is the duck-typed view of pii.block_unscanned_media +// (satisfied by config.ModelConfig). +type unscannedMediaPolicy interface { + PIIBlockUnscannedMedia() bool +} + +// unscannedMediaPattern is the sentinel PatternID recorded when a request is +// blocked because it carries content the detectors cannot inspect. +const unscannedMediaPattern = "__unscanned_media__" + // RequestMiddleware applies the regex PII tier to incoming chat // requests. If the parsed request is not a MessageScanner (e.g., // non-chat endpoints registered against the same group later), the @@ -161,6 +176,15 @@ func RequestMiddleware(redactor *Redactor, store EventStore, adapter Adapter, fa return next(c) } + // Content the text detectors cannot read (images, audio, files) + // would otherwise pass unchecked. When the model asks for it, + // refuse such requests before any detector runs. + if pol, ok := rawCfg.(unscannedMediaPolicy); ok && pol.PIIBlockUnscannedMedia() && adapter.Unscanned != nil { + if kinds := adapter.Unscanned(parsed); len(kinds) > 0 { + return blockUnscannedMedia(c, store, kinds, fallbackUser) + } + } + // A PII-enabled model with no detectors (or no resolver wired) // has nothing to scan with — pass through. if len(detectors) == 0 || o.nerResolver == nil { @@ -307,6 +331,46 @@ func RequestMiddleware(redactor *Redactor, store EventStore, adapter Adapter, fa } } +// blockUnscannedMedia records an audit event and rejects the request with the +// same 400 pii_blocked shape as a content block: from the client's point of +// view the request carried something the policy does not let through. +func blockUnscannedMedia(c echo.Context, store EventStore, kinds []string, fallbackUser *auth.User) error { + user := auth.GetUser(c) + if user == nil { + user = fallbackUser + } + userID := "" + if user != nil { + userID = user.ID + } + correlationID, _ := c.Get(ctxKeyCorrelationID).(string) + ev := PIIEvent{ + ID: newEventID(), + Kind: KindPII, + Origin: OriginMiddleware, + CorrelationID: correlationID, + UserID: userID, + Direction: DirectionIn, + PatternID: unscannedMediaPattern, + Action: ActionBlock, + CreatedAt: time.Now().UTC(), + } + if store != nil { + if err := store.Record(context.Background(), ev); err != nil { + xlog.Error("pii: failed to record event", "error", err, "pattern", unscannedMediaPattern) + } + } + xlog.Info("pii: request blocked, content cannot be inspected", "kinds", kinds) + return c.JSON(http.StatusBadRequest, map[string]any{ + "error": map[string]string{ + "message": "request blocked by content policy (content that cannot be inspected: " + strings.Join(kinds, ", ") + ")", + "type": "pii_blocked", + }, + "correlation_id": correlationID, + "pii_event_id": ev.ID, + }) +} + // nerUnavailablePattern is the sentinel PatternID recorded on the // fail-closed audit event when a model's configured NER tier cannot // run. It is not a real regex pattern — it marks a request blocked diff --git a/core/services/routing/pii/unscanned_media_test.go b/core/services/routing/pii/unscanned_media_test.go new file mode 100644 index 000000000000..b802d100bfe6 --- /dev/null +++ b/core/services/routing/pii/unscanned_media_test.go @@ -0,0 +1,100 @@ +package pii + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + + "github.com/labstack/echo/v4" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// mediaPIIConfig adds pii.block_unscanned_media to the fake model config. +type mediaPIIConfig struct { + fakeModelPIIConfig + blockMedia bool +} + +func (m mediaPIIConfig) PIIBlockUnscannedMedia() bool { return m.blockMedia } + +// serveAny is serve() for a config value of any type. +func serveAny(body *fakeRequest, cfg any, mw echo.MiddlewareFunc) (*httptest.ResponseRecorder, *bool) { + called := new(bool) + e := echo.New() + setCfg := func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + c.Set(ctxKeyModelConfig, cfg) + return next(c) + } + } + e.POST("/chat", func(c echo.Context) error { + *called = true + return c.JSON(http.StatusOK, map[string]string{"ok": "yes"}) + }, setRequestOnContext(body), setCfg, mw) + w := httptest.NewRecorder() + e.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/chat", strings.NewReader(`{}`))) + return w, called +} + +func mediaAdapter(kinds ...string) Adapter { + a := fakeAdapter() + a.Unscanned = func(any) []string { return kinds } + return a +} + +var _ = Describe("RequestMiddleware (block_unscanned_media)", func() { + clean := map[string]NERConfig{"privacy-filter": nerCfg(ActionBlock)} + + run := func(cfg mediaPIIConfig, adapter Adapter, st EventStore) (int, map[string]any, bool) { + body := &fakeRequest{Messages: []string{"what is on this picture?"}} + mw := RequestMiddleware(&Redactor{}, st, adapter, nil, WithNERResolver(resolverFor(clean))) + w, called := serveAny(body, cfg, mw) + var out map[string]any + _ = json.Unmarshal(w.Body.Bytes(), &out) + return w.Code, out, *called + } + + It("blocks a request with media the detectors cannot inspect and records an event", func() { + st := NewMemoryEventStore(0) + code, out, called := run(mediaPIIConfig{fakeModelPIIConfig{enabled: true, detectors: []string{"privacy-filter"}}, true}, + mediaAdapter("image_url"), st) + Expect(code).To(Equal(http.StatusBadRequest)) + Expect(called).To(BeFalse(), "the handler (and so the upstream) must not see the request") + Expect(out["error"].(map[string]any)["type"]).To(Equal("pii_blocked")) + Expect(out["error"].(map[string]any)["message"]).To(ContainSubstring("image_url")) + events, _ := st.List(context.Background(), ListQuery{Limit: 10}) + Expect(events).To(HaveLen(1)) + Expect(events[0].PatternID).To(Equal(unscannedMediaPattern)) + Expect(events[0].Action).To(Equal(ActionBlock)) + }) + + It("blocks even when the model lists no detectors", func() { + code, _, called := run(mediaPIIConfig{fakeModelPIIConfig{enabled: true}, true}, mediaAdapter("input_audio"), nil) + Expect(code).To(Equal(http.StatusBadRequest)) + Expect(called).To(BeFalse()) + }) + + It("lets text-only requests through", func() { + code, _, called := run(mediaPIIConfig{fakeModelPIIConfig{enabled: true, detectors: []string{"privacy-filter"}}, true}, + mediaAdapter(), nil) + Expect(code).To(Equal(http.StatusOK)) + Expect(called).To(BeTrue()) + }) + + It("does not block media when the option is off (default)", func() { + code, _, called := run(mediaPIIConfig{fakeModelPIIConfig{enabled: true, detectors: []string{"privacy-filter"}}, false}, + mediaAdapter("image_url"), nil) + Expect(code).To(Equal(http.StatusOK)) + Expect(called).To(BeTrue()) + }) + + It("does not block media when PII is disabled for the model", func() { + code, _, called := run(mediaPIIConfig{fakeModelPIIConfig{enabled: false}, true}, mediaAdapter("image_url"), nil) + Expect(code).To(Equal(http.StatusOK)) + Expect(called).To(BeTrue()) + }) +}) diff --git a/core/services/routing/piiadapter/anthropic.go b/core/services/routing/piiadapter/anthropic.go index e059e6bc1881..5fca14be0f74 100644 --- a/core/services/routing/piiadapter/anthropic.go +++ b/core/services/routing/piiadapter/anthropic.go @@ -18,6 +18,11 @@ import ( // System field, not in Messages — they're skipped here for now (chat // messages are the high-traffic surface). System-prompt scanning is a // follow-up if a deployment proves it needs it. +// anthropicTextBlocks are the message block types that carry no media of their +// own (tool calls and results are structured text). +var anthropicTextBlocks = map[string]bool{"text": true, "tool_use": true, "tool_result": true, + "thinking": true, "redacted_thinking": true} + func Anthropic() pii.Adapter { return pii.Adapter{ Scan: func(parsed any) []pii.ScannedText { @@ -53,6 +58,20 @@ func Anthropic() pii.Adapter { } return out }, + Unscanned: func(parsed any) []string { + req, ok := parsed.(*schema.AnthropicRequest) + if !ok || req == nil { + return nil + } + var out []string + seen := map[string]bool{} + for i := range req.Messages { + if blocks, ok := req.Messages[i].Content.([]any); ok { + out = blockKinds(blocks, anthropicTextBlocks, seen, out) + } + } + return out + }, Apply: func(parsed any, updates []pii.ScannedText) { req, ok := parsed.(*schema.AnthropicRequest) if !ok || req == nil { diff --git a/core/services/routing/piiadapter/ollama.go b/core/services/routing/piiadapter/ollama.go index f837cc9a0cb9..de6a7792f189 100644 --- a/core/services/routing/piiadapter/ollama.go +++ b/core/services/routing/piiadapter/ollama.go @@ -23,6 +23,18 @@ func OllamaChat() pii.Adapter { } return out }, + Unscanned: func(parsed any) []string { + req, ok := parsed.(*schema.OllamaChatRequest) + if !ok || req == nil { + return nil + } + for i := range req.Messages { + if len(req.Messages[i].Images) > 0 { + return []string{"images"} + } + } + return nil + }, Apply: func(parsed any, updates []pii.ScannedText) { req, ok := parsed.(*schema.OllamaChatRequest) if !ok || req == nil { diff --git a/core/services/routing/piiadapter/openai.go b/core/services/routing/piiadapter/openai.go index ee668ba694a3..df26462bdeed 100644 --- a/core/services/routing/piiadapter/openai.go +++ b/core/services/routing/piiadapter/openai.go @@ -25,6 +25,9 @@ import ( // System / developer / tool messages are scanned as well: an API key // pasted into a system prompt is just as leak-prone as one in a user // message. +// openAITextBlocks are the chat content block types that carry only text. +var openAITextBlocks = map[string]bool{"text": true, "refusal": true} + func OpenAI() pii.Adapter { return pii.Adapter{ Scan: func(parsed any) []pii.ScannedText { @@ -64,6 +67,20 @@ func OpenAI() pii.Adapter { } return out }, + Unscanned: func(parsed any) []string { + req, ok := parsed.(*schema.OpenAIRequest) + if !ok || req == nil { + return nil + } + var out []string + seen := map[string]bool{} + for i := range req.Messages { + if blocks, ok := req.Messages[i].Content.([]any); ok { + out = blockKinds(blocks, openAITextBlocks, seen, out) + } + } + return out + }, Apply: func(parsed any, updates []pii.ScannedText) { req, ok := parsed.(*schema.OpenAIRequest) if !ok || req == nil { diff --git a/core/services/routing/piiadapter/unscanned.go b/core/services/routing/piiadapter/unscanned.go new file mode 100644 index 000000000000..ec327748a5d0 --- /dev/null +++ b/core/services/routing/piiadapter/unscanned.go @@ -0,0 +1,20 @@ +package piiadapter + +// blockKinds collects, in order and without duplicates, the "type" of every +// structured content block that is not text, i.e. content the text detectors +// never see. textTypes lists the block types that carry no media of their own. +func blockKinds(blocks []any, textTypes map[string]bool, seen map[string]bool, out []string) []string { + for _, block := range blocks { + m, ok := block.(map[string]any) + if !ok { + continue + } + t, _ := m["type"].(string) + if t == "" || textTypes[t] || seen[t] { + continue + } + seen[t] = true + out = append(out, t) + } + return out +} diff --git a/core/services/routing/piiadapter/unscanned_test.go b/core/services/routing/piiadapter/unscanned_test.go new file mode 100644 index 000000000000..865f06b249c3 --- /dev/null +++ b/core/services/routing/piiadapter/unscanned_test.go @@ -0,0 +1,58 @@ +package piiadapter + +import ( + "github.com/mudler/LocalAI/core/schema" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Unscanned content", func() { + It("lists OpenAI media block types once, in order", func() { + req := &schema.OpenAIRequest{Messages: []schema.Message{ + {Role: "user", Content: []any{ + map[string]any{"type": "text", "text": "what is this?"}, + map[string]any{"type": "image_url", "image_url": map[string]any{"url": "data:image/png;base64,xyz"}}, + }}, + {Role: "user", Content: []any{ + map[string]any{"type": "input_audio", "input_audio": map[string]any{"data": "abc"}}, + map[string]any{"type": "image_url", "image_url": map[string]any{"url": "https://example.com/a.png"}}, + map[string]any{"type": "file", "file": map[string]any{"file_data": "JVBERi0="}}, + }}, + }} + Expect(OpenAI().Unscanned(req)).To(Equal([]string{"image_url", "input_audio", "file"})) + }) + + It("reports nothing for OpenAI text-only requests", func() { + req := &schema.OpenAIRequest{Messages: []schema.Message{ + {Role: "system", Content: "be brief"}, + {Role: "user", Content: []any{map[string]any{"type": "text", "text": "hi"}}}, + }} + Expect(OpenAI().Unscanned(req)).To(BeEmpty()) + }) + + It("lists Anthropic image and document blocks but not tool blocks", func() { + req := &schema.AnthropicRequest{Messages: []schema.AnthropicMessage{ + {Role: "user", Content: []any{ + map[string]any{"type": "text", "text": "read this"}, + map[string]any{"type": "document", "source": map[string]any{"type": "base64"}}, + }}, + {Role: "assistant", Content: []any{map[string]any{"type": "tool_use", "name": "x"}}}, + {Role: "user", Content: []any{ + map[string]any{"type": "tool_result", "content": "ok"}, + map[string]any{"type": "image", "source": map[string]any{"type": "base64"}}, + }}, + }} + Expect(Anthropic().Unscanned(req)).To(Equal([]string{"document", "image"})) + textOnly := &schema.AnthropicRequest{Messages: []schema.AnthropicMessage{{Role: "user", Content: "hi"}}} + Expect(Anthropic().Unscanned(textOnly)).To(BeEmpty()) + }) + + It("reports Ollama chat images", func() { + req := &schema.OllamaChatRequest{Messages: []schema.OllamaMessage{ + {Role: "user", Content: "what is this?", Images: []string{"aGVsbG8="}}, + }} + Expect(OllamaChat().Unscanned(req)).To(Equal([]string{"images"})) + textOnly := &schema.OllamaChatRequest{Messages: []schema.OllamaMessage{{Role: "user", Content: "hi"}}} + Expect(OllamaChat().Unscanned(textOnly)).To(BeEmpty()) + }) +}) diff --git a/docs/content/operations/middleware.md b/docs/content/operations/middleware.md index 7f8bb769ee24..541cc81cdfc6 100644 --- a/docs/content/operations/middleware.md +++ b/docs/content/operations/middleware.md @@ -271,6 +271,15 @@ Reversible redactions provide less confidentiality than irreversible masking: any third party that can observe both the redacted request and restored response may be able to infer the original values. +`block_unscanned_media: true` closes the gap the text detectors leave open: +images, audio, video and files in a request are never scanned, so by default +they reach the backend unchecked. With the option set, a request that carries +any such content is rejected with HTTP 400 (`error.type=pii_blocked`, audit +event pattern `__unscanned_media__`) before a detector runs. Use it for egress +policies where "cannot be inspected" must mean "not sent", e.g. on +`cloud-proxy` models. It applies to the OpenAI chat, Anthropic messages and +Ollama chat routes, and only while PII is enabled for the model. + Multiple detectors **union** their detections; overlapping spans resolve to the strongest action (`block` > `mask` > `allow`). A configured detector that can't be loaded **fails the request closed** (HTTP 503,