diff --git a/core/config/meta/registry.go b/core/config/meta/registry.go index ea900d79e5ad..f89c0d3f95ea 100644 --- a/core/config/meta/registry.go +++ b/core/config/meta/registry.go @@ -1008,6 +1008,13 @@ func DefaultRegistry() map[string]FieldMetaOverride { Description: "Suffix for reversible redaction tokens. Defaults to ].", Order: 204, }, + "pii.block_unscanned_media": { + Section: "pii", + Label: "Block Unscanned Media", + Description: "Reject requests that carry content the detectors cannot inspect (images, audio, video, files) instead of forwarding it unchecked. Only applies while PII is enabled for this model.", + Component: "toggle", + Order: 205, + }, // --- PII detection policy (on a token_classify detector model) --- "pii_detection.min_score": { diff --git a/core/config/model_config.go b/core/config/model_config.go index a52618fd2351..247b97093085 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -622,11 +622,19 @@ type PIIConfig struct { ReversibleRedactions bool `yaml:"reversible_redactions,omitempty" json:"reversible_redactions,omitempty"` ReversibleTokenPrefix string `yaml:"reversible_token_prefix,omitempty" json:"reversible_token_prefix,omitempty"` ReversibleTokenSuffix string `yaml:"reversible_token_suffix,omitempty" json:"reversible_token_suffix,omitempty"` + + // BlockUnscannedMedia rejects requests that carry content the text + // detectors cannot inspect (images, audio, video, files) instead of + // forwarding it unchecked. Only applies while PII is enabled for the + // model. Meant for egress policies where "not inspectable" must mean + // "not sent", e.g. cloud-proxy models. + BlockUnscannedMedia bool `yaml:"block_unscanned_media,omitempty" json:"block_unscanned_media,omitempty"` } func (c ModelConfig) PIIReversibleRedactions() bool { return c.PII.ReversibleRedactions } func (c ModelConfig) PIIReversibleTokenPrefix() string { return c.PII.ReversibleTokenPrefix } func (c ModelConfig) PIIReversibleTokenSuffix() string { return c.PII.ReversibleTokenSuffix } +func (c ModelConfig) PIIBlockUnscannedMedia() bool { return c.PII.BlockUnscannedMedia } // @Description Detection policy for a token-classification (NER) model // used as a PII detector. Lives on the detector model's own config so the diff --git a/core/services/routing/pii/middleware.go b/core/services/routing/pii/middleware.go index bf77d916d4c0..17aa8ffdc492 100644 --- a/core/services/routing/pii/middleware.go +++ b/core/services/routing/pii/middleware.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "encoding/hex" "net/http" + "strings" "time" "github.com/labstack/echo/v4" @@ -107,8 +108,22 @@ type ScannedText struct { type Adapter struct { Scan func(parsed any) []ScannedText Apply func(parsed any, updates []ScannedText) + // Unscanned lists the kinds of content in the request that Scan does + // not return as text (e.g. "image_url", "input_file"). Optional; used + // only when the model sets pii.block_unscanned_media. + Unscanned func(parsed any) []string } +// unscannedMediaPolicy is the duck-typed view of pii.block_unscanned_media +// (satisfied by config.ModelConfig). +type unscannedMediaPolicy interface { + PIIBlockUnscannedMedia() bool +} + +// unscannedMediaPattern is the sentinel PatternID recorded when a request is +// blocked because it carries content the detectors cannot inspect. +const unscannedMediaPattern = "__unscanned_media__" + // RequestMiddleware applies the regex PII tier to incoming chat // requests. If the parsed request is not a MessageScanner (e.g., // non-chat endpoints registered against the same group later), the @@ -161,6 +176,15 @@ func RequestMiddleware(redactor *Redactor, store EventStore, adapter Adapter, fa return next(c) } + // Content the text detectors cannot read (images, audio, files) + // would otherwise pass unchecked. When the model asks for it, + // refuse such requests before any detector runs. + if pol, ok := rawCfg.(unscannedMediaPolicy); ok && pol.PIIBlockUnscannedMedia() && adapter.Unscanned != nil { + if kinds := adapter.Unscanned(parsed); len(kinds) > 0 { + return blockUnscannedMedia(c, store, kinds, fallbackUser) + } + } + // A PII-enabled model with no detectors (or no resolver wired) // has nothing to scan with — pass through. if len(detectors) == 0 || o.nerResolver == nil { @@ -307,6 +331,46 @@ func RequestMiddleware(redactor *Redactor, store EventStore, adapter Adapter, fa } } +// blockUnscannedMedia records an audit event and rejects the request with the +// same 400 pii_blocked shape as a content block: from the client's point of +// view the request carried something the policy does not let through. +func blockUnscannedMedia(c echo.Context, store EventStore, kinds []string, fallbackUser *auth.User) error { + user := auth.GetUser(c) + if user == nil { + user = fallbackUser + } + userID := "" + if user != nil { + userID = user.ID + } + correlationID, _ := c.Get(ctxKeyCorrelationID).(string) + ev := PIIEvent{ + ID: newEventID(), + Kind: KindPII, + Origin: OriginMiddleware, + CorrelationID: correlationID, + UserID: userID, + Direction: DirectionIn, + PatternID: unscannedMediaPattern, + Action: ActionBlock, + CreatedAt: time.Now().UTC(), + } + if store != nil { + if err := store.Record(context.Background(), ev); err != nil { + xlog.Error("pii: failed to record event", "error", err, "pattern", unscannedMediaPattern) + } + } + xlog.Info("pii: request blocked, content cannot be inspected", "kinds", kinds) + return c.JSON(http.StatusBadRequest, map[string]any{ + "error": map[string]string{ + "message": "request blocked by content policy (content that cannot be inspected: " + strings.Join(kinds, ", ") + ")", + "type": "pii_blocked", + }, + "correlation_id": correlationID, + "pii_event_id": ev.ID, + }) +} + // nerUnavailablePattern is the sentinel PatternID recorded on the // fail-closed audit event when a model's configured NER tier cannot // run. It is not a real regex pattern — it marks a request blocked diff --git a/core/services/routing/pii/unscanned_media_test.go b/core/services/routing/pii/unscanned_media_test.go new file mode 100644 index 000000000000..b802d100bfe6 --- /dev/null +++ b/core/services/routing/pii/unscanned_media_test.go @@ -0,0 +1,100 @@ +package pii + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + + "github.com/labstack/echo/v4" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// mediaPIIConfig adds pii.block_unscanned_media to the fake model config. +type mediaPIIConfig struct { + fakeModelPIIConfig + blockMedia bool +} + +func (m mediaPIIConfig) PIIBlockUnscannedMedia() bool { return m.blockMedia } + +// serveAny is serve() for a config value of any type. +func serveAny(body *fakeRequest, cfg any, mw echo.MiddlewareFunc) (*httptest.ResponseRecorder, *bool) { + called := new(bool) + e := echo.New() + setCfg := func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + c.Set(ctxKeyModelConfig, cfg) + return next(c) + } + } + e.POST("/chat", func(c echo.Context) error { + *called = true + return c.JSON(http.StatusOK, map[string]string{"ok": "yes"}) + }, setRequestOnContext(body), setCfg, mw) + w := httptest.NewRecorder() + e.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/chat", strings.NewReader(`{}`))) + return w, called +} + +func mediaAdapter(kinds ...string) Adapter { + a := fakeAdapter() + a.Unscanned = func(any) []string { return kinds } + return a +} + +var _ = Describe("RequestMiddleware (block_unscanned_media)", func() { + clean := map[string]NERConfig{"privacy-filter": nerCfg(ActionBlock)} + + run := func(cfg mediaPIIConfig, adapter Adapter, st EventStore) (int, map[string]any, bool) { + body := &fakeRequest{Messages: []string{"what is on this picture?"}} + mw := RequestMiddleware(&Redactor{}, st, adapter, nil, WithNERResolver(resolverFor(clean))) + w, called := serveAny(body, cfg, mw) + var out map[string]any + _ = json.Unmarshal(w.Body.Bytes(), &out) + return w.Code, out, *called + } + + It("blocks a request with media the detectors cannot inspect and records an event", func() { + st := NewMemoryEventStore(0) + code, out, called := run(mediaPIIConfig{fakeModelPIIConfig{enabled: true, detectors: []string{"privacy-filter"}}, true}, + mediaAdapter("image_url"), st) + Expect(code).To(Equal(http.StatusBadRequest)) + Expect(called).To(BeFalse(), "the handler (and so the upstream) must not see the request") + Expect(out["error"].(map[string]any)["type"]).To(Equal("pii_blocked")) + Expect(out["error"].(map[string]any)["message"]).To(ContainSubstring("image_url")) + events, _ := st.List(context.Background(), ListQuery{Limit: 10}) + Expect(events).To(HaveLen(1)) + Expect(events[0].PatternID).To(Equal(unscannedMediaPattern)) + Expect(events[0].Action).To(Equal(ActionBlock)) + }) + + It("blocks even when the model lists no detectors", func() { + code, _, called := run(mediaPIIConfig{fakeModelPIIConfig{enabled: true}, true}, mediaAdapter("input_audio"), nil) + Expect(code).To(Equal(http.StatusBadRequest)) + Expect(called).To(BeFalse()) + }) + + It("lets text-only requests through", func() { + code, _, called := run(mediaPIIConfig{fakeModelPIIConfig{enabled: true, detectors: []string{"privacy-filter"}}, true}, + mediaAdapter(), nil) + Expect(code).To(Equal(http.StatusOK)) + Expect(called).To(BeTrue()) + }) + + It("does not block media when the option is off (default)", func() { + code, _, called := run(mediaPIIConfig{fakeModelPIIConfig{enabled: true, detectors: []string{"privacy-filter"}}, false}, + mediaAdapter("image_url"), nil) + Expect(code).To(Equal(http.StatusOK)) + Expect(called).To(BeTrue()) + }) + + It("does not block media when PII is disabled for the model", func() { + code, _, called := run(mediaPIIConfig{fakeModelPIIConfig{enabled: false}, true}, mediaAdapter("image_url"), nil) + Expect(code).To(Equal(http.StatusOK)) + Expect(called).To(BeTrue()) + }) +}) diff --git a/core/services/routing/piiadapter/anthropic.go b/core/services/routing/piiadapter/anthropic.go index e059e6bc1881..5fca14be0f74 100644 --- a/core/services/routing/piiadapter/anthropic.go +++ b/core/services/routing/piiadapter/anthropic.go @@ -18,6 +18,11 @@ import ( // System field, not in Messages — they're skipped here for now (chat // messages are the high-traffic surface). System-prompt scanning is a // follow-up if a deployment proves it needs it. +// anthropicTextBlocks are the message block types that carry no media of their +// own (tool calls and results are structured text). +var anthropicTextBlocks = map[string]bool{"text": true, "tool_use": true, "tool_result": true, + "thinking": true, "redacted_thinking": true} + func Anthropic() pii.Adapter { return pii.Adapter{ Scan: func(parsed any) []pii.ScannedText { @@ -53,6 +58,20 @@ func Anthropic() pii.Adapter { } return out }, + Unscanned: func(parsed any) []string { + req, ok := parsed.(*schema.AnthropicRequest) + if !ok || req == nil { + return nil + } + var out []string + seen := map[string]bool{} + for i := range req.Messages { + if blocks, ok := req.Messages[i].Content.([]any); ok { + out = blockKinds(blocks, anthropicTextBlocks, seen, out) + } + } + return out + }, Apply: func(parsed any, updates []pii.ScannedText) { req, ok := parsed.(*schema.AnthropicRequest) if !ok || req == nil { diff --git a/core/services/routing/piiadapter/ollama.go b/core/services/routing/piiadapter/ollama.go index f837cc9a0cb9..de6a7792f189 100644 --- a/core/services/routing/piiadapter/ollama.go +++ b/core/services/routing/piiadapter/ollama.go @@ -23,6 +23,18 @@ func OllamaChat() pii.Adapter { } return out }, + Unscanned: func(parsed any) []string { + req, ok := parsed.(*schema.OllamaChatRequest) + if !ok || req == nil { + return nil + } + for i := range req.Messages { + if len(req.Messages[i].Images) > 0 { + return []string{"images"} + } + } + return nil + }, Apply: func(parsed any, updates []pii.ScannedText) { req, ok := parsed.(*schema.OllamaChatRequest) if !ok || req == nil { diff --git a/core/services/routing/piiadapter/openai.go b/core/services/routing/piiadapter/openai.go index ee668ba694a3..df26462bdeed 100644 --- a/core/services/routing/piiadapter/openai.go +++ b/core/services/routing/piiadapter/openai.go @@ -25,6 +25,9 @@ import ( // System / developer / tool messages are scanned as well: an API key // pasted into a system prompt is just as leak-prone as one in a user // message. +// openAITextBlocks are the chat content block types that carry only text. +var openAITextBlocks = map[string]bool{"text": true, "refusal": true} + func OpenAI() pii.Adapter { return pii.Adapter{ Scan: func(parsed any) []pii.ScannedText { @@ -64,6 +67,20 @@ func OpenAI() pii.Adapter { } return out }, + Unscanned: func(parsed any) []string { + req, ok := parsed.(*schema.OpenAIRequest) + if !ok || req == nil { + return nil + } + var out []string + seen := map[string]bool{} + for i := range req.Messages { + if blocks, ok := req.Messages[i].Content.([]any); ok { + out = blockKinds(blocks, openAITextBlocks, seen, out) + } + } + return out + }, Apply: func(parsed any, updates []pii.ScannedText) { req, ok := parsed.(*schema.OpenAIRequest) if !ok || req == nil { diff --git a/core/services/routing/piiadapter/unscanned.go b/core/services/routing/piiadapter/unscanned.go new file mode 100644 index 000000000000..ec327748a5d0 --- /dev/null +++ b/core/services/routing/piiadapter/unscanned.go @@ -0,0 +1,20 @@ +package piiadapter + +// blockKinds collects, in order and without duplicates, the "type" of every +// structured content block that is not text, i.e. content the text detectors +// never see. textTypes lists the block types that carry no media of their own. +func blockKinds(blocks []any, textTypes map[string]bool, seen map[string]bool, out []string) []string { + for _, block := range blocks { + m, ok := block.(map[string]any) + if !ok { + continue + } + t, _ := m["type"].(string) + if t == "" || textTypes[t] || seen[t] { + continue + } + seen[t] = true + out = append(out, t) + } + return out +} diff --git a/core/services/routing/piiadapter/unscanned_test.go b/core/services/routing/piiadapter/unscanned_test.go new file mode 100644 index 000000000000..865f06b249c3 --- /dev/null +++ b/core/services/routing/piiadapter/unscanned_test.go @@ -0,0 +1,58 @@ +package piiadapter + +import ( + "github.com/mudler/LocalAI/core/schema" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Unscanned content", func() { + It("lists OpenAI media block types once, in order", func() { + req := &schema.OpenAIRequest{Messages: []schema.Message{ + {Role: "user", Content: []any{ + map[string]any{"type": "text", "text": "what is this?"}, + map[string]any{"type": "image_url", "image_url": map[string]any{"url": "data:image/png;base64,xyz"}}, + }}, + {Role: "user", Content: []any{ + map[string]any{"type": "input_audio", "input_audio": map[string]any{"data": "abc"}}, + map[string]any{"type": "image_url", "image_url": map[string]any{"url": "https://example.com/a.png"}}, + map[string]any{"type": "file", "file": map[string]any{"file_data": "JVBERi0="}}, + }}, + }} + Expect(OpenAI().Unscanned(req)).To(Equal([]string{"image_url", "input_audio", "file"})) + }) + + It("reports nothing for OpenAI text-only requests", func() { + req := &schema.OpenAIRequest{Messages: []schema.Message{ + {Role: "system", Content: "be brief"}, + {Role: "user", Content: []any{map[string]any{"type": "text", "text": "hi"}}}, + }} + Expect(OpenAI().Unscanned(req)).To(BeEmpty()) + }) + + It("lists Anthropic image and document blocks but not tool blocks", func() { + req := &schema.AnthropicRequest{Messages: []schema.AnthropicMessage{ + {Role: "user", Content: []any{ + map[string]any{"type": "text", "text": "read this"}, + map[string]any{"type": "document", "source": map[string]any{"type": "base64"}}, + }}, + {Role: "assistant", Content: []any{map[string]any{"type": "tool_use", "name": "x"}}}, + {Role: "user", Content: []any{ + map[string]any{"type": "tool_result", "content": "ok"}, + map[string]any{"type": "image", "source": map[string]any{"type": "base64"}}, + }}, + }} + Expect(Anthropic().Unscanned(req)).To(Equal([]string{"document", "image"})) + textOnly := &schema.AnthropicRequest{Messages: []schema.AnthropicMessage{{Role: "user", Content: "hi"}}} + Expect(Anthropic().Unscanned(textOnly)).To(BeEmpty()) + }) + + It("reports Ollama chat images", func() { + req := &schema.OllamaChatRequest{Messages: []schema.OllamaMessage{ + {Role: "user", Content: "what is this?", Images: []string{"aGVsbG8="}}, + }} + Expect(OllamaChat().Unscanned(req)).To(Equal([]string{"images"})) + textOnly := &schema.OllamaChatRequest{Messages: []schema.OllamaMessage{{Role: "user", Content: "hi"}}} + Expect(OllamaChat().Unscanned(textOnly)).To(BeEmpty()) + }) +}) diff --git a/docs/content/operations/middleware.md b/docs/content/operations/middleware.md index 7f8bb769ee24..541cc81cdfc6 100644 --- a/docs/content/operations/middleware.md +++ b/docs/content/operations/middleware.md @@ -271,6 +271,15 @@ Reversible redactions provide less confidentiality than irreversible masking: any third party that can observe both the redacted request and restored response may be able to infer the original values. +`block_unscanned_media: true` closes the gap the text detectors leave open: +images, audio, video and files in a request are never scanned, so by default +they reach the backend unchecked. With the option set, a request that carries +any such content is rejected with HTTP 400 (`error.type=pii_blocked`, audit +event pattern `__unscanned_media__`) before a detector runs. Use it for egress +policies where "cannot be inspected" must mean "not sent", e.g. on +`cloud-proxy` models. It applies to the OpenAI chat, Anthropic messages and +Ollama chat routes, and only while PII is enabled for the model. + Multiple detectors **union** their detections; overlapping spans resolve to the strongest action (`block` > `mask` > `allow`). A configured detector that can't be loaded **fails the request closed** (HTTP 503,