TAB-976 Harden form fills against prompt context exfiltration

packages/core/src/browser/ariaBrowser.ts

@@ -57,6 +57,39 @@ export interface TemporaryTab {
   waitForLoadState(state: LoadState, options?: { timeout?: number }): Promise<void>;
 }
 
+export interface FieldMetadata {
+  ref: string;
+  tagName: string;
+  inputType: string | null;
+  role: string | null;
+  name: string | null;
+  label: string | null;
+  placeholder: string | null;
+  autocomplete: string | null;
+  isContentEditable: boolean;
+  formId: string | null;
+  formAction: string | null;
+  formMethod: string | null;
+}
+
+export interface FormFieldState {
+  ref: string | null;
+  name: string | null;
+  tagName: string;
+  inputType: string | null;
+  autocomplete: string | null;
+}
+
+export interface FormSubmissionContext {
+  submitterRef: string;
+  formId: string | null;
+  actionUrl: string | null;
+  method: string | null;
+  fields: FormFieldState[];
+}
+
+export type FormSubmissionTrigger = "click" | "enter";
+
 export interface AriaBrowser {
   /** The name of the browser being used */
   browserName: string;
@@ -99,6 +132,15 @@ export interface AriaBrowser {
    */
   performAction(ref: string, action: PageAction, value?: string): Promise<void>;
 
+  /** Returns structural metadata for an element ref used in form/action policy checks. */
+  getFieldMetadata(ref: string): Promise<FieldMetadata>;
+
+  /** Returns the form that would be submitted by activating this ref, if any. */
+  getFormSubmissionContext(
+    ref: string,
+    trigger?: FormSubmissionTrigger,
+  ): Promise<FormSubmissionContext | null>;
+
   /**
    * Waits for a specific load state of the page
    * @param state The load state to wait for

packages/core/src/browser/playwrightBrowser.ts

@@ -11,7 +11,15 @@ import {
   Locator,
   errors as playwrightErrors,
 } from "playwright";
-import { AriaBrowser, PageAction, LoadState, TemporaryTab } from "./ariaBrowser.js";
+import {
+  AriaBrowser,
+  FieldMetadata,
+  FormSubmissionTrigger,
+  FormSubmissionContext,
+  LoadState,
+  PageAction,
+  TemporaryTab,
+} from "./ariaBrowser.js";
 import { PlaywrightBlocker } from "@ghostery/adblocker-playwright";
 import fetch from "cross-fetch";
 import TurndownService from "turndown";
@@ -788,6 +796,181 @@ export class PlaywrightBrowser implements AriaBrowser {
     return locator;
   }
 
+  async getFieldMetadata(ref: string): Promise<FieldMetadata> {
+    const locator = await this.validateElementRef(ref);
+
+    return locator.evaluate((element, elementRef): FieldMetadata => {
+      const el = element as HTMLElement;
+      const input = el instanceof HTMLInputElement ? el : null;
+      const form = getElementForm(el);
+
+      return {
+        ref: elementRef,
+        tagName: el.tagName.toLowerCase(),
+        inputType: input?.type?.toLowerCase() ?? null,
+        role: el.getAttribute("role"),
+        name: getElementName(el),
+        label: getElementLabel(el),
+        placeholder: getElementPlaceholder(el),
+        autocomplete: getElementAutocomplete(el),
+        isContentEditable: el.isContentEditable,
+        formId: form?.id || null,
+        formAction: form?.action || null,
+        formMethod: form?.method?.toLowerCase() || null,
+      };
+
+      function getElementForm(node: HTMLElement): HTMLFormElement | null {
+        if (
+          node instanceof HTMLInputElement ||
+          node instanceof HTMLTextAreaElement ||
+          node instanceof HTMLSelectElement ||
+          node instanceof HTMLButtonElement
+        ) {
+          return node.form;
+        }
+        return node.closest("form");
+      }
+
+      function getElementName(node: HTMLElement): string | null {
+        if (
+          node instanceof HTMLInputElement ||
+          node instanceof HTMLTextAreaElement ||
+          node instanceof HTMLSelectElement ||
+          node instanceof HTMLButtonElement
+        ) {
+          return node.name || null;
+        }
+        return node.getAttribute("name");
+      }
+
+      function getElementLabel(node: HTMLElement): string | null {
+        const ariaLabel = node.getAttribute("aria-label");
+        if (ariaLabel?.trim()) return ariaLabel.trim();
+
+        const labelledBy = node.getAttribute("aria-labelledby");
+        if (labelledBy) {
+          const text = labelledBy
+            .split(/\s+/)
+            .map((id) => node.ownerDocument.getElementById(id)?.textContent?.trim() || "")
+            .filter(Boolean)
+            .join(" ");
+          if (text) return text;
+        }
+
+        if ("labels" in node) {
+          const labels = (node as HTMLInputElement | HTMLTextAreaElement | HTMLSelectElement)
+            .labels;
+          const text = Array.from(labels || [])
+            .map((label) => label.textContent?.trim() || "")
+            .filter(Boolean)
+            .join(" ");
+          if (text) return text;
+        }
+
+        return null;
+      }
+
+      function getElementPlaceholder(node: HTMLElement): string | null {
+        if (node instanceof HTMLInputElement || node instanceof HTMLTextAreaElement) {
+          return node.placeholder || null;
+        }
+        return null;
+      }
+
+      function getElementAutocomplete(node: HTMLElement): string | null {
+        if (
+          node instanceof HTMLInputElement ||
+          node instanceof HTMLTextAreaElement ||
+          node instanceof HTMLSelectElement
+        ) {
+          return node.autocomplete || null;
+        }
+        return null;
+      }
+    }, ref);
+  }
+
+  async getFormSubmissionContext(
+    ref: string,
+    trigger: FormSubmissionTrigger = "click",
+  ): Promise<FormSubmissionContext | null> {
+    const locator = await this.validateElementRef(ref);
+
+    return locator.evaluate(
+      (element, { submitterRef, trigger }): FormSubmissionContext | null => {
+        const el = element as HTMLElement;
+        if (!canSubmitForm(el, trigger)) return null;
+
+        const form = getSubmissionForm(el);
+        if (!form) return null;
+
+        const fields = Array.from(form.elements)
+          .filter(
+            (field): field is HTMLInputElement | HTMLTextAreaElement | HTMLSelectElement =>
+              field instanceof HTMLInputElement ||
+              field instanceof HTMLTextAreaElement ||
+              field instanceof HTMLSelectElement,
+          )
+          .filter((field) => !field.disabled)
+          .map((field) => ({
+            ref: field.getAttribute("data-pilo-ref"),
+            name: field.name || null,
+            tagName: field.tagName.toLowerCase(),
+            inputType: field instanceof HTMLInputElement ? field.type.toLowerCase() : null,
+            autocomplete: "autocomplete" in field ? field.autocomplete || null : null,
+          }));
+
+        return {
+          submitterRef,
+          formId: form.id || null,
+          actionUrl: form.action || null,
+          method: form.method?.toLowerCase() || null,
+          fields,
+        };
+
+        function getSubmissionForm(node: HTMLElement): HTMLFormElement | null {
+          if (
+            node instanceof HTMLButtonElement ||
+            node instanceof HTMLInputElement ||
+            node instanceof HTMLTextAreaElement ||
+            node instanceof HTMLSelectElement
+          ) {
+            return node.form;
+          }
+          return node.closest("form");
+        }
+
+        function canSubmitForm(node: HTMLElement, submitTrigger: FormSubmissionTrigger): boolean {
+          if (submitTrigger === "click") {
+            if (node instanceof HTMLButtonElement) {
+              return node.type === "submit";
+            }
+            if (node instanceof HTMLInputElement) {
+              return node.type === "submit" || node.type === "image";
+            }
+            return false;
+          }
+
+          if (node instanceof HTMLTextAreaElement || node instanceof HTMLSelectElement)
+            return false;
+          if (!(node instanceof HTMLInputElement)) return false;
+          return ![
+            "button",
+            "checkbox",
+            "color",
+            "file",
+            "hidden",
+            "radio",
+            "range",
+            "reset",
+            "submit",
+          ].includes(node.type);
+        }
+      },
+      { submitterRef: ref, trigger },
+    );
+  }
+
   async performAction(ref: string, action: PageAction, value?: string): Promise<void> {
     if (!this.page) throw new Error("Browser not started");
     return withSpan(

packages/core/src/core.ts

@@ -5,7 +5,12 @@
  */
 
 export { WebAgent } from "./webAgent.js";
-export type { AriaBrowser } from "./browser/ariaBrowser.js";
+export type {
+  AriaBrowser,
+  FieldMetadata,
+  FormSubmissionContext,
+  FormSubmissionTrigger,
+} from "./browser/ariaBrowser.js";
 export { PageAction, LoadState } from "./browser/ariaBrowser.js";
 export type { TaskExecutionResult, TaskError, WebAgentOptions } from "./webAgent.js";
 export { TaskErrorCode } from "./webAgent.js";