diff --git a/evaluators/README.md b/evaluators/README.md index 53ec15d3..a179b2be 100644 --- a/evaluators/README.md +++ b/evaluators/README.md @@ -19,9 +19,14 @@ Pattern matching for text (PII, keywords, SQL injection) ``` ### List -Match against value lists (blocked users, restricted cities) +Match against value lists (blocked users, restricted cities, allowed prefixes). +Supports `match_mode: "exact"` for full-string membership, `match_mode: "contains"` +for keyword-style matching, `match_mode: "starts_with"` for prefix matching, and +`match_mode: "ends_with"` for suffix matching. ```python {"name": "list", "config": {"values": ["admin", "root"], "case_sensitive": False}} +{"name": "list", "config": {"values": ["/home/lev/agent-control"], "match_mode": "starts_with"}} +{"name": "list", "config": {"values": [".md"], "match_mode": "ends_with"}} ``` ### SQL diff --git a/evaluators/builtin/src/agent_control_evaluators/list/config.py b/evaluators/builtin/src/agent_control_evaluators/list/config.py index 43ae664c..a6f323d2 100644 --- a/evaluators/builtin/src/agent_control_evaluators/list/config.py +++ b/evaluators/builtin/src/agent_control_evaluators/list/config.py @@ -19,9 +19,12 @@ class ListEvaluatorConfig(EvaluatorConfig): match_on: Literal["match", "no_match"] = Field( "match", description="Trigger rule on match or no match" ) - match_mode: Literal["exact", "contains"] = Field( + match_mode: Literal["exact", "contains", "starts_with", "ends_with"] = Field( "exact", - description="'exact' for full string match, 'contains' for keyword/substring match", + description=( + "'exact' for full string match, 'contains' for keyword matching, " + "'starts_with' for prefix matching, and 'ends_with' for suffix matching" + ), ) case_sensitive: bool = Field(False, description="Whether matching is case sensitive") diff --git a/evaluators/builtin/src/agent_control_evaluators/list/evaluator.py b/evaluators/builtin/src/agent_control_evaluators/list/evaluator.py index 55bcd78b..27ecc0dc 100644 --- a/evaluators/builtin/src/agent_control_evaluators/list/evaluator.py +++ b/evaluators/builtin/src/agent_control_evaluators/list/evaluator.py @@ -18,7 +18,7 @@ class ListEvaluator(Evaluator[ListEvaluatorConfig]): Checks if data matches values in a list. Supports: - any/all logic (match any value vs match all values) - match/no_match trigger (trigger on match or no match) - - exact/contains mode (full match vs substring/keyword) + - exact/contains/starts_with/ends_with mode (full match vs keyword vs prefix/suffix) - case sensitivity toggle Example configs: @@ -50,6 +50,12 @@ def _build_regex(self) -> Any: if self.config.match_mode == "contains": # Word boundary matching for substring/keyword detection pattern = f"\\b({'|'.join(escaped)})\\b" + elif self.config.match_mode == "starts_with": + # Prefix matching using anchors + pattern = f"^({'|'.join(escaped)})" + elif self.config.match_mode == "ends_with": + # Suffix matching using anchors + pattern = f"({'|'.join(escaped)})$" else: # Exact match using anchors pattern = f"^({'|'.join(escaped)})$" diff --git a/evaluators/builtin/tests/list/test_list.py b/evaluators/builtin/tests/list/test_list.py index 81ca2818..3ee950d0 100644 --- a/evaluators/builtin/tests/list/test_list.py +++ b/evaluators/builtin/tests/list/test_list.py @@ -33,6 +33,439 @@ def test_whitespace_only_value_rejected(self) -> None: class TestListEvaluator: """Tests for list evaluator runtime behavior.""" + @pytest.mark.asyncio + async def test_starts_with_matches_prefix(self) -> None: + """Test that starts_with mode triggers on prefix matches.""" + # Given: a starts_with evaluator config + evaluator = ListEvaluator( + ListEvaluatorConfig( + values=["/home/lev/agent-control", "/tmp/cache"], + logic="any", + match_on="match", + match_mode="starts_with", + case_sensitive=True, + ) + ) + + # When: evaluating a path under an allowed prefix + result = await evaluator.evaluate("/home/lev/agent-control/server/src/app.py") + + # Then: the prefix match triggers + assert result.matched is True + assert result.metadata["matches"] == ["/home/lev/agent-control/server/src/app.py"] + + @pytest.mark.asyncio + async def test_starts_with_matches_exact_path_value(self) -> None: + """Test that starts_with mode matches the configured path value exactly.""" + # Given: a starts_with evaluator config + evaluator = ListEvaluator( + ListEvaluatorConfig( + values=["/home/lev/agent-control"], + logic="any", + match_on="match", + match_mode="starts_with", + case_sensitive=True, + ) + ) + + # When: evaluating the exact configured path + result = await evaluator.evaluate("/home/lev/agent-control") + + # Then: the exact path matches + assert result.matched is True + assert result.metadata["matches"] == ["/home/lev/agent-control"] + + @pytest.mark.asyncio + async def test_starts_with_no_match_when_prefix_absent(self) -> None: + """Test that starts_with mode does not trigger when no prefix matches.""" + # Given: a starts_with evaluator config + evaluator = ListEvaluator( + ListEvaluatorConfig( + values=["/home/lev/agent-control", "/tmp/cache"], + logic="any", + match_on="match", + match_mode="starts_with", + case_sensitive=True, + ) + ) + + # When: evaluating a path with no configured prefix + result = await evaluator.evaluate("/var/log/system.log") + + # Then: the evaluator does not trigger + assert result.matched is False + + @pytest.mark.asyncio + async def test_starts_with_uses_raw_string_prefix_for_path_like_values(self) -> None: + """Test that starts_with is generic string-prefix matching, not path-segment aware.""" + # Given: a starts_with evaluator configured with a path-like prefix + evaluator = ListEvaluator( + ListEvaluatorConfig( + values=["/home/lev/agent-control"], + logic="any", + match_on="match", + match_mode="starts_with", + case_sensitive=True, + ) + ) + + # When: evaluating a sibling path that shares the same string prefix + result = await evaluator.evaluate("/home/lev/agent-control-old/server") + + # Then: the evaluator matches because starts_with is not path-boundary aware + assert result.matched is True + assert result.metadata["matches"] == ["/home/lev/agent-control-old/server"] + + @pytest.mark.asyncio + async def test_starts_with_honors_case_sensitivity(self) -> None: + """Test that starts_with mode respects case sensitivity settings.""" + # Given: two starts_with evaluators that differ only by case sensitivity + insensitive = ListEvaluator( + ListEvaluatorConfig( + values=["/HOME/LEV/AGENT-CONTROL"], + logic="any", + match_on="match", + match_mode="starts_with", + case_sensitive=False, + ) + ) + sensitive = ListEvaluator( + ListEvaluatorConfig( + values=["/HOME/LEV/AGENT-CONTROL"], + logic="any", + match_on="match", + match_mode="starts_with", + case_sensitive=True, + ) + ) + + # When: evaluating the same lower-case path against both + insensitive_result = await insensitive.evaluate("/home/lev/agent-control/server") + sensitive_result = await sensitive.evaluate("/home/lev/agent-control/server") + + # Then: only the case-insensitive evaluator matches + assert insensitive_result.matched is True + assert sensitive_result.matched is False + + @pytest.mark.asyncio + async def test_starts_with_supports_no_match_allowlists(self) -> None: + """Test that starts_with works with no_match for allowlist-style controls.""" + # Given: a starts_with evaluator configured as an allowlist + evaluator = ListEvaluator( + ListEvaluatorConfig( + values=["/home/lev/agent-control", "/tmp/cache"], + logic="any", + match_on="no_match", + match_mode="starts_with", + case_sensitive=True, + ) + ) + + # When: evaluating one allowed and one disallowed path + allowed_result = await evaluator.evaluate("/home/lev/agent-control/server") + denied_result = await evaluator.evaluate("/var/log/system.log") + + # Then: only the disallowed path triggers the control + assert allowed_result.matched is False + assert denied_result.matched is True + + @pytest.mark.asyncio + async def test_starts_with_matches_plain_text_prefix(self) -> None: + """Test that starts_with mode works for ordinary non-path strings.""" + # Given: a starts_with evaluator config for ordinary strings + evaluator = ListEvaluator( + ListEvaluatorConfig( + values=["agent", "control:"], + logic="any", + match_on="match", + match_mode="starts_with", + case_sensitive=True, + ) + ) + + # When: evaluating values with and without the configured prefixes + matched_result = await evaluator.evaluate("agent-control") + unmatched_result = await evaluator.evaluate("please control: now") + + # Then: only the true prefix match triggers + assert matched_result.matched is True + assert unmatched_result.matched is False + + @pytest.mark.asyncio + async def test_starts_with_escapes_regex_metacharacters(self) -> None: + """Test that starts_with treats configured values as literals, not regex.""" + # Given: prefixes containing regex metacharacters + evaluator = ListEvaluator( + ListEvaluatorConfig( + values=["release/v1.2+", "[beta]"], + logic="any", + match_on="match", + match_mode="starts_with", + case_sensitive=True, + ) + ) + + # When: evaluating values that begin with those literal prefixes + release_result = await evaluator.evaluate("release/v1.2+rc1") + beta_result = await evaluator.evaluate("[beta] feature flag") + + # Then: both values match literally + assert release_result.matched is True + assert beta_result.matched is True + + @pytest.mark.asyncio + async def test_starts_with_supports_list_input_with_any_logic(self) -> None: + """Test that starts_with works on list inputs when any item matches.""" + # Given: a starts_with evaluator with any-item semantics + evaluator = ListEvaluator( + ListEvaluatorConfig( + values=["/home/lev/agent-control", "agent"], + logic="any", + match_on="match", + match_mode="starts_with", + case_sensitive=True, + ) + ) + + # When: evaluating a list where only one element matches + result = await evaluator.evaluate(["/var/log/system.log", "agent-control"]) + + # Then: the evaluator triggers and reports the matching entry + assert result.matched is True + assert result.metadata["matches"] == ["agent-control"] + + @pytest.mark.asyncio + async def test_starts_with_supports_list_input_with_all_logic(self) -> None: + """Test that starts_with respects all-item semantics for list inputs.""" + # Given: a starts_with evaluator with all-item semantics + evaluator = ListEvaluator( + ListEvaluatorConfig( + values=["/home/lev/agent-control", "/tmp/cache"], + logic="all", + match_on="match", + match_mode="starts_with", + case_sensitive=True, + ) + ) + + # When: evaluating one fully matching list and one partially matching list + matching_result = await evaluator.evaluate( + ["/home/lev/agent-control/server", "/tmp/cache/build"] + ) + partial_result = await evaluator.evaluate( + ["/home/lev/agent-control/server", "/var/log/system.log"] + ) + + # Then: only the fully matching list triggers + assert matching_result.matched is True + assert partial_result.matched is False + + @pytest.mark.asyncio + async def test_ends_with_matches_suffix(self) -> None: + """Test that ends_with mode triggers on suffix matches.""" + # Given: an ends_with evaluator config + evaluator = ListEvaluator( + ListEvaluatorConfig( + values=["/SOUL.md", ".py"], + logic="any", + match_on="match", + match_mode="ends_with", + case_sensitive=True, + ) + ) + + # When: evaluating a path with an allowed suffix + result = await evaluator.evaluate("/home/lev/agent-control/SOUL.md") + + # Then: the suffix match triggers + assert result.matched is True + assert result.metadata["matches"] == ["/home/lev/agent-control/SOUL.md"] + + @pytest.mark.asyncio + async def test_ends_with_matches_exact_path_value(self) -> None: + """Test that ends_with mode matches the configured path value exactly.""" + # Given: an ends_with evaluator config + evaluator = ListEvaluator( + ListEvaluatorConfig( + values=["/home/lev/agent-control/SOUL.md"], + logic="any", + match_on="match", + match_mode="ends_with", + case_sensitive=True, + ) + ) + + # When: evaluating the exact configured path + result = await evaluator.evaluate("/home/lev/agent-control/SOUL.md") + + # Then: the exact path matches + assert result.matched is True + assert result.metadata["matches"] == ["/home/lev/agent-control/SOUL.md"] + + @pytest.mark.asyncio + async def test_ends_with_no_match_when_suffix_absent(self) -> None: + """Test that ends_with mode does not trigger when no suffix matches.""" + # Given: an ends_with evaluator config + evaluator = ListEvaluator( + ListEvaluatorConfig( + values=["/SOUL.md", ".py"], + logic="any", + match_on="match", + match_mode="ends_with", + case_sensitive=True, + ) + ) + + # When: evaluating a path with no configured suffix + result = await evaluator.evaluate("/var/log/system.log") + + # Then: the evaluator does not trigger + assert result.matched is False + + @pytest.mark.asyncio + async def test_ends_with_honors_case_sensitivity(self) -> None: + """Test that ends_with mode respects case sensitivity settings.""" + # Given: two ends_with evaluators that differ only by case sensitivity + insensitive = ListEvaluator( + ListEvaluatorConfig( + values=["/SOUL.MD"], + logic="any", + match_on="match", + match_mode="ends_with", + case_sensitive=False, + ) + ) + sensitive = ListEvaluator( + ListEvaluatorConfig( + values=["/SOUL.MD"], + logic="any", + match_on="match", + match_mode="ends_with", + case_sensitive=True, + ) + ) + + # When: evaluating the same lower-case path against both + insensitive_result = await insensitive.evaluate("/home/lev/agent-control/SOUL.md") + sensitive_result = await sensitive.evaluate("/home/lev/agent-control/SOUL.md") + + # Then: only the case-insensitive evaluator matches + assert insensitive_result.matched is True + assert sensitive_result.matched is False + + @pytest.mark.asyncio + async def test_ends_with_supports_no_match_allowlists(self) -> None: + """Test that ends_with works with no_match for allowlist-style controls.""" + # Given: an ends_with evaluator configured as an allowlist + evaluator = ListEvaluator( + ListEvaluatorConfig( + values=[".md", ".txt"], + logic="any", + match_on="no_match", + match_mode="ends_with", + case_sensitive=True, + ) + ) + + # When: evaluating one allowed and one disallowed path + allowed_result = await evaluator.evaluate("/home/lev/agent-control/SOUL.md") + denied_result = await evaluator.evaluate("/var/log/system.log") + + # Then: only the disallowed path triggers the control + assert allowed_result.matched is False + assert denied_result.matched is True + + @pytest.mark.asyncio + async def test_ends_with_matches_plain_text_suffix(self) -> None: + """Test that ends_with mode works for ordinary non-path strings.""" + # Given: an ends_with evaluator config for ordinary strings + evaluator = ListEvaluator( + ListEvaluatorConfig( + values=["-control", ":done"], + logic="any", + match_on="match", + match_mode="ends_with", + case_sensitive=True, + ) + ) + + # When: evaluating values with and without the configured suffixes + matched_result = await evaluator.evaluate("agent-control") + unmatched_result = await evaluator.evaluate("done: please") + + # Then: only the true suffix match triggers + assert matched_result.matched is True + assert unmatched_result.matched is False + + @pytest.mark.asyncio + async def test_ends_with_escapes_regex_metacharacters(self) -> None: + """Test that ends_with treats configured values as literals, not regex.""" + # Given: suffixes containing regex metacharacters + evaluator = ListEvaluator( + ListEvaluatorConfig( + values=[".tar.gz+", "[beta]"], + logic="any", + match_on="match", + match_mode="ends_with", + case_sensitive=True, + ) + ) + + # When: evaluating values that end with those literal suffixes + archive_result = await evaluator.evaluate("release-1.tar.gz+") + beta_result = await evaluator.evaluate("feature-[beta]") + + # Then: both values match literally + assert archive_result.matched is True + assert beta_result.matched is True + + @pytest.mark.asyncio + async def test_ends_with_supports_list_input_with_any_logic(self) -> None: + """Test that ends_with works on list inputs when any item matches.""" + # Given: an ends_with evaluator with any-item semantics + evaluator = ListEvaluator( + ListEvaluatorConfig( + values=[".md", "-control"], + logic="any", + match_on="match", + match_mode="ends_with", + case_sensitive=True, + ) + ) + + # When: evaluating a list where only one element matches + result = await evaluator.evaluate(["/var/log/system.log", "agent-control"]) + + # Then: the evaluator triggers and reports the matching entry + assert result.matched is True + assert result.metadata["matches"] == ["agent-control"] + + @pytest.mark.asyncio + async def test_ends_with_supports_list_input_with_all_logic(self) -> None: + """Test that ends_with respects all-item semantics for list inputs.""" + # Given: an ends_with evaluator with all-item semantics + evaluator = ListEvaluator( + ListEvaluatorConfig( + values=[".md", ".txt"], + logic="all", + match_on="match", + match_mode="ends_with", + case_sensitive=True, + ) + ) + + # When: evaluating one fully matching list and one partially matching list + matching_result = await evaluator.evaluate( + ["/home/lev/agent-control/SOUL.md", "/tmp/cache/notes.txt"] + ) + partial_result = await evaluator.evaluate( + ["/home/lev/agent-control/SOUL.md", "/var/log/system.log"] + ) + + # Then: only the fully matching list triggers + assert matching_result.matched is True + assert partial_result.matched is False + @pytest.mark.asyncio async def test_legacy_empty_string_value_is_ignored_defensively(self) -> None: """Test that legacy invalid configs do not compile into a match-all regex.""" diff --git a/ui/src/core/evaluators/list/form.tsx b/ui/src/core/evaluators/list/form.tsx index 025178f9..8aee4bc8 100644 --- a/ui/src/core/evaluators/list/form.tsx +++ b/ui/src/core/evaluators/list/form.tsx @@ -75,13 +75,15 @@ export const ListForm = ({ form }: EvaluatorFormProps) => { label={ } labelProps={labelPropsInline} data={[ { value: 'exact', label: 'Exact (full string match)' }, - { value: 'contains', label: 'Contains (substring match)' }, + { value: 'contains', label: 'Contains (keyword match)' }, + { value: 'starts_with', label: 'Starts with (prefix match)' }, + { value: 'ends_with', label: 'Ends with (suffix match)' }, ]} size="sm" {...form.getInputProps('match_mode')} diff --git a/ui/src/core/evaluators/list/types.ts b/ui/src/core/evaluators/list/types.ts index 4d5e191c..1f7505d4 100644 --- a/ui/src/core/evaluators/list/types.ts +++ b/ui/src/core/evaluators/list/types.ts @@ -9,8 +9,8 @@ export type ListFormValues = { logic: 'any' | 'all'; /** When to trigger: on match or no match */ match_on: 'match' | 'no_match'; - /** Match mode: exact or contains */ - match_mode: 'exact' | 'contains'; + /** Match mode: exact, contains, starts_with, or ends_with */ + match_mode: 'exact' | 'contains' | 'starts_with' | 'ends_with'; /** Whether matching is case sensitive */ case_sensitive: boolean; };