Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
200 changes: 197 additions & 3 deletions src/agents/planner-executor/planner-executor-agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -892,6 +892,7 @@ export class PlannerExecutorAgent {
}
}

plannerAction = this.promoteVisibleResultClick(task, ctx, plannerAction);
this.composableHeuristics.setStepHints(plannerAction.heuristicHints || []);
this.emitPlannerAction(stepNum, plannerAction, plannerActionSource);

Expand Down Expand Up @@ -1182,6 +1183,20 @@ export class PlannerExecutorAgent {
};
}

if (this.isCopiedPlaceholderNavigation(plannerAction.target, currentUrl, task)) {
return {
stepId: stepNum,
goal: stepGoal,
status: StepStatus.SKIPPED,
actionTaken: 'SKIPPED(placeholder_navigation)',
verificationPassed: true,
usedVision: false,
durationMs: Date.now() - stepStart,
urlBefore: currentUrl,
urlAfter: currentUrl,
};
}

try {
await runtime.goto(plannerAction.target);
const verificationPassed = await this.verifyStepOutcome(runtime, plannerAction);
Expand Down Expand Up @@ -1414,10 +1429,29 @@ export class PlannerExecutorAgent {
const elementId = parsed.args[0] as number;

if (parsed.action === 'CLICK') {
const targetElement =
activeCtx.snapshot?.elements.find(element => element.id === elementId) || null;
await runtime.click(elementId);
await this.handlePostClickEffects(runtime, plannerAction, activeCtx);
const verificationPassed = await this.verifyStepOutcome(runtime, plannerAction);
const urlAfter = await runtime.getCurrentUrl();
const hasUrlVerification = (plannerAction.verify || []).some(
predicate =>
predicate.predicate === 'url_contains' ||
predicate.predicate === 'url_equals' ||
predicate.predicate === 'url_matches'
);
const relevantUrlChange = isUrlChangeRelevantToIntent(
currentUrl,
urlAfter,
plannerAction,
targetElement
);
const navigationSatisfied =
relevantUrlChange &&
(!hasUrlVerification ||
this.clickedHrefMatchesNavigation(currentUrl, urlAfter, targetElement));
const verificationPassed =
navigationSatisfied || (await this.verifyStepOutcome(runtime, plannerAction));
return {
stepId: stepNum,
goal: plannerAction.intent || 'Click element',
Expand All @@ -1437,6 +1471,7 @@ export class PlannerExecutorAgent {
const elements = activeCtx.snapshot?.elements || [];
const inputElement = elements.find(element => element.id === elementId) || null;
const isSearchLike = isSearchLikeTypeAndSubmit(plannerAction, inputElement);
let submissionSatisfied = false;

// Submit with Enter key for TYPE_AND_SUBMIT, plus planner TYPE actions that clearly target search.
if (
Expand All @@ -1448,7 +1483,6 @@ export class PlannerExecutorAgent {
const hasRetryBudget = this.config.retry.executorRepairAttempts > 0;

let changedUrl: string | null = null;
let submissionSatisfied = false;

const checkSubmissionSatisfied = async (): Promise<boolean> => {
if (
Expand Down Expand Up @@ -1535,7 +1569,8 @@ export class PlannerExecutorAgent {
}
}

const verificationPassed = await this.verifyStepOutcome(runtime, plannerAction);
const verificationPassed =
submissionSatisfied || (await this.verifyStepOutcome(runtime, plannerAction));
const urlAfter = await runtime.getCurrentUrl();

return {
Expand Down Expand Up @@ -2055,6 +2090,127 @@ export class PlannerExecutorAgent {
};
}

private isCopiedPlaceholderNavigation(
targetUrl: string,
currentUrl: string,
task: string
): boolean {
if (!this.isExampleDotComUrl(targetUrl)) {
return false;
}

if (this.isExampleDotComUrl(currentUrl) || /\bexample\.com\b/i.test(task)) {
return false;
}

return true;
}

private isExampleDotComUrl(url: string): boolean {
try {
const hostname = new URL(url).hostname.toLowerCase();
return hostname === 'example.com' || hostname.endsWith('.example.com');
} catch {
return false;
}
}

private promoteVisibleResultClick(
task: string,
ctx: SnapshotContext,
plannerAction: StepwisePlannerResponse
): StepwisePlannerResponse {
if (plannerAction.action !== 'SCROLL' && plannerAction.action !== 'WAIT') {
return plannerAction;
}

const candidate = this.findVisibleResultLink(task, ctx.snapshot);
if (!candidate) {
return plannerAction;
}

const label = this.elementLabel(candidate);
const hrefVerify = this.hrefVerificationSignal(candidate.href || '', ctx.snapshot?.url || '');

return {
...plannerAction,
action: 'CLICK',
goal: plannerAction.goal || 'Open visible result link',
intent: 'visible result link',
input: label || plannerAction.input,
verify: hrefVerify ? [{ predicate: 'url_contains', args: [hrefVerify] }] : [],
heuristicHints: [
{
intent_pattern: 'visible_result_link',
text_patterns: label ? [label] : [],
role_filter: ['link'],
priority: 20,
},
],
reasoning:
plannerAction.reasoning ||
'Visible result link matched the task goal; clicking it is more direct than scrolling.',
};
}

private findVisibleResultLink(
task: string,
snapshot: Snapshot | null | undefined
): SnapshotElement | null {
const elements = snapshot?.elements || [];
if (elements.length === 0 || !this.taskWantsResultNavigation(task)) {
return null;
}

const candidates = elements
.filter(element => this.isResultNavigationLink(element))
.sort((left, right) => (right.importance || 0) - (left.importance || 0));

return candidates[0] || null;
}

private taskWantsResultNavigation(task: string): boolean {
const normalized = task.toLowerCase().replace(/[_-]+/g, ' ').replace(/\s+/g, ' ');
const actionCue = /\b(click|open|pick|choose|select|go to|visit)\b/.test(normalized);
const targetCue = /\b(product|result|item|listing|detail page|details page)\b/.test(normalized);
return actionCue && targetCue;
}

private isResultNavigationLink(element: SnapshotElement): boolean {
const role = (element.role || '').toLowerCase();
const href = (element.href || '').toLowerCase();
if (role !== 'link' || !href) {
return false;
}

if (/\/(?:dp|gp\/product|product|products|item|items|p)\//.test(href)) {
return true;
}

const label = this.elementLabel(element);
return Boolean(element.inDominantGroup && label.length >= 15);
}

private elementLabel(element: SnapshotElement): string {
return (element.text || element.ariaLabel || element.name || '').trim();
}

private hrefVerificationSignal(href: string, baseUrl: string): string | null {
if (!href.trim()) {
return null;
}

try {
const parsed = new URL(href, baseUrl || undefined);
if (parsed.pathname && parsed.pathname !== '/') {
return parsed.pathname;
}
return parsed.hostname || null;
} catch {
return href.startsWith('/') ? href : null;
}
}

private summarizePlannerActionTarget(plannerAction: StepwisePlannerResponse): string | null {
if (plannerAction.action === 'TYPE' || plannerAction.action === 'TYPE_AND_SUBMIT') {
return plannerAction.input || plannerAction.intent || plannerAction.target || null;
Expand Down Expand Up @@ -2287,6 +2443,44 @@ export class PlannerExecutorAgent {
return false;
}

private clickedHrefMatchesNavigation(
previousUrl: string,
nextUrl: string,
element: SnapshotElement | null
): boolean {
const href = element?.href?.trim();
if (!href) {
return false;
}

try {
const expected = new URL(href, previousUrl);
const actual = new URL(nextUrl, previousUrl);
if (
!['http:', 'https:'].includes(expected.protocol) ||
!['http:', 'https:'].includes(actual.protocol)
) {
return false;
}

expected.hash = '';
actual.hash = '';
expected.search = '';
actual.search = '';

return (
this.normalizeNavigationUrl(expected.toString()) ===
this.normalizeNavigationUrl(actual.toString())
);
} catch {
return false;
}
}

private normalizeNavigationUrl(url: string): string {
return url.trim().replace(/\/+$/, '').toLowerCase();
}

private async isCartAdditionTerminal(
runtime: AgentRuntime,
task: string,
Expand Down
5 changes: 3 additions & 2 deletions src/agents/planner-executor/prompts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ CRITICAL RULE FOR ADD TO CART:
- Set "input" to "Add to Cart" (or exact button text from elements)

Output ONLY valid JSON (no markdown, no \`\`\`):
{"action":"NAVIGATE","target":"https://example.com/search","verify":[{"predicate":"url_contains","args":["search"]}],"reasoning":"open the search page"}
{"action":"NAVIGATE","target":"https://shop.test/search","verify":[{"predicate":"url_contains","args":["search"]}],"reasoning":"open the known search page"}
{"action":"TYPE_AND_SUBMIT","intent":"searchbox","input":"wireless headphones","verify":[{"predicate":"url_contains","args":["search"]}],"reasoning":"search for product"}
{"action":"CLICK","intent":"product link","input":"Sony WH-1000XM4 Wireless...","verify":[],"required":true,"heuristic_hints":[{"intent_pattern":"product_link","text_patterns":["sony wh-1000xm4"],"role_filter":["link"],"priority":8}],"reasoning":"click first product result"}
{"action":"CLICK","intent":"add to cart button","input":"Add to Cart","verify":[],"required":true,"heuristic_hints":[{"intent_pattern":"add_to_cart","text_patterns":["add to cart","buy now"],"role_filter":["button"],"priority":10}],"reasoning":"add item to cart"}
Expand All @@ -95,7 +95,8 @@ RULES:
7. "heuristic_hints" entries may use snake_case fields: "intent_pattern", "text_patterns", "role_filter", "attribute_patterns", "priority"
8. Output ONLY JSON - no <think> tags, no markdown, no prose
9. Do NOT output <think> or any reasoning
10. Do NOT return DONE until ALL parts of the goal are complete`;
10. Do NOT return DONE until ALL parts of the goal are complete
11. Never copy example URLs from these instructions. Only NAVIGATE to a URL from the user's task, the current page, or a visible element.`;

// NOTE: /no_think MUST be at the START of user message for Qwen3 models
const user = `/no_think
Expand Down
Loading
Loading