Skip to content

Commit 433fd0f

Browse files
committed
test(evals): assert grounded behavior instead of exact phrasing in live mode
The first live DeepSeek run exposed brittle assertions, not harness bugs: the model chained the tools correctly but the checks were case-sensitive and required an internal order id. Match the retrieved value case-insensitively and let live runs accept the grounded status rather than the internal id.
1 parent af2e6ca commit 433fd0f

1 file changed

Lines changed: 4 additions & 2 deletions

File tree

‎apps/sim/evals/agent-tool-use/scenarios.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,7 @@ export const AGENT_TOOL_USE_SCENARIOS: AgentToolUseScenario[] = [
182182
],
183183
expect: {
184184
toolCallSequence: ['list_files', 'read_file'],
185-
finalContent: 'ship the eval harness',
185+
finalContent: /ship the eval harness/i,
186186
maxIterations: 4,
187187
successfulToolCalls: 2,
188188
},
@@ -216,6 +216,8 @@ export const AGENT_TOOL_USE_SCENARIOS: AgentToolUseScenario[] = [
216216
finalContent: /A-1937.*shipped/,
217217
maxIterations: 3,
218218
},
219+
/** A live model may answer with the user-facing order number and the grounded status. */
220+
liveExpect: { finalContent: /shipped/i },
219221
},
220222
{
221223
id: 'parallel-independent-tools',
@@ -245,7 +247,7 @@ export const AGENT_TOOL_USE_SCENARIOS: AgentToolUseScenario[] = [
245247
],
246248
expect: {
247249
toolCallSequence: ['get_weather', 'get_news'],
248-
finalContent: /12°C.*transit strike ends/,
250+
finalContent: /12°C[\s\S]*transit strike ends/i,
249251
maxIterations: 3,
250252
successfulToolCalls: 2,
251253
},

0 commit comments

Comments
 (0)