Skip to content

Commit cdebc4d

Browse files
committed
test(evals): fix adversarial scenario design after the live run
Two live failures were eval design, not model failure: - empty-result-no-hallucination rejected valid 'didn't find' / 'wasn't able to find' phrasing. Broaden the grounding check. - near-duplicate-names required a userId the prompt never gave, so the model reasonably asked for it. Put the id in the prompt and the scripted call.
1 parent 9d2ec19 commit cdebc4d

1 file changed

Lines changed: 3 additions & 3 deletions

File tree

‎apps/sim/evals/agent-tool-use/scenarios.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -499,7 +499,7 @@ export const AGENT_TOOL_USE_SCENARIOS: AgentToolUseScenario[] = [
499499
expect: {
500500
requiredTools: ['search_docs'],
501501
finalContent:
502-
/could ?n[o']t find|cannot find|can'?t find|no results|not find|unable to find|no .*policy/i,
502+
/could ?n'?o?t find|cannot find|can'?t find|didn'?t find|wasn'?t able|not able to find|unable to find|no results|no matching|no documentation|no information|not covered|returned empty|came back empty|found nothing|no .*policy/i,
503503
maxIterations: 3,
504504
},
505505
},
@@ -567,15 +567,15 @@ export const AGENT_TOOL_USE_SCENARIOS: AgentToolUseScenario[] = [
567567
category: 'tool-selection',
568568
description:
569569
'get_user and get_user_settings differ by suffix; only the settings tool has the timezone.',
570-
userMessage: 'What timezone is set in my account settings?',
570+
userMessage: 'What timezone is set in my account settings? My user id is u-42.',
571571
tools: [getUser, getUserSettings],
572572
script: [
573573
{
574574
kind: 'tools',
575575
calls: [
576576
{
577577
name: 'get_user_settings',
578-
args: { userId: 'me' },
578+
args: { userId: 'u-42' },
579579
result: { success: true, output: { timezone: 'Europe/Berlin' } },
580580
},
581581
],

0 commit comments

Comments
 (0)