From 2be2bffad97cbeb6ff5562df832ef2b06717de50 Mon Sep 17 00:00:00 2001 From: RichardHruby Date: Tue, 17 Feb 2026 20:35:01 -0800 Subject: [PATCH] feat: add social login support and noop screen type Co-Authored-By: Claude Opus 4.6 --- src/app/api/chat/route.ts | 7 ++++- src/components/chat.tsx | 2 ++ src/components/credential-form.tsx | 34 ++++++++++++++++++++++- src/components/message-bubble.tsx | 44 +++++++++++++++++++++++++++--- src/hooks/use-login-session.ts | 21 +++++++++++++- src/lib/ai-login/agent.ts | 39 ++++++++++++++++++++++++++ src/lib/ai-login/prompts.ts | 17 ++++++++++-- src/lib/ai-login/types.ts | 22 +++++++++++++++ 8 files changed, 177 insertions(+), 9 deletions(-) diff --git a/src/app/api/chat/route.ts b/src/app/api/chat/route.ts index 612890c..4bd0e38 100644 --- a/src/app/api/chat/route.ts +++ b/src/app/api/chat/route.ts @@ -150,7 +150,12 @@ export async function POST(request: NextRequest) { const hasUserInput = Object.values(values).some((v) => v); const isUserAction = hasUserInput && - ["credential_login_form", "choice_screen", "magic_login_link"].includes( + [ + "credential_login_form", + "choice_screen", + "magic_login_link", + "noop_screen", + ].includes( screen.type, ); diff --git a/src/components/chat.tsx b/src/components/chat.tsx index e9c2dbd..12703ea 100644 --- a/src/components/chat.tsx +++ b/src/components/chat.tsx @@ -43,6 +43,7 @@ export function Chat() { busy, logs, formStatuses, + formSubmitLabels, activeFormId, startSession, submitForm, @@ -157,6 +158,7 @@ export function Chat() { busy={busy} activeFormId={activeFormId} formStatuses={formStatuses} + formSubmitLabels={formSubmitLabels} showLabel={i === 0 || messages[i - 1].role !== msg.role} /> ))} diff --git a/src/components/credential-form.tsx b/src/components/credential-form.tsx index 77d9dbc..183b3cf 100644 --- a/src/components/credential-form.tsx +++ b/src/components/credential-form.tsx @@ -14,13 +14,18 @@ import { Lock } from "lucide-react"; import { Input } from "@/components/ui/input"; import { Button } from "@/components/ui/button"; import { Label } from "@/components/ui/label"; -import type { InputElementType } from "@/lib/ai-login/types"; +import type { + InputElementType, + SocialLoginOptionType, +} from "@/lib/ai-login/types"; import { cn } from "@/lib/utils"; interface CredentialFormProps { inputs: InputElementType[]; submitLabel: string; onSubmit: (values: Record) => void; + socialLogins?: SocialLoginOptionType[]; + onSocialLogin?: (provider: string) => void; disabled?: boolean; } @@ -56,6 +61,8 @@ export function CredentialForm({ inputs, submitLabel, onSubmit, + socialLogins, + onSocialLogin, disabled, }: CredentialFormProps) { const [values, setValues] = useState>({}); @@ -123,6 +130,31 @@ export function CredentialForm({ {submitLabel} + + {/* Social login buttons */} + {socialLogins && socialLogins.length > 0 && onSocialLogin && ( +
+
+
+ or +
+
+
+ {socialLogins.map((sl) => ( + + ))} +
+
+ )} ); } diff --git a/src/components/message-bubble.tsx b/src/components/message-bubble.tsx index dff5ede..831101d 100644 --- a/src/components/message-bubble.tsx +++ b/src/components/message-bubble.tsx @@ -13,6 +13,7 @@ import { Loader2, Check, LockOpen } from "lucide-react"; import type { ChatMessage, FormStatus } from "@/hooks/use-login-session"; import type { LoginState } from "@/lib/ai-login/types"; +import { Button } from "@/components/ui/button"; import { CredentialForm } from "./credential-form"; import { ChoiceButtons } from "./choice-buttons"; import { MagicLinkInput } from "./magic-link-input"; @@ -23,6 +24,7 @@ interface MessageBubbleProps { busy: boolean; activeFormId: string | null; formStatuses: Record; + formSubmitLabels: Record; showLabel?: boolean; } @@ -32,6 +34,7 @@ export function MessageBubble({ busy, activeFormId, formStatuses, + formSubmitLabels, showLabel, }: MessageBubbleProps) { // User message @@ -125,7 +128,11 @@ export function MessageBubble({ return (
{assistantLabel} - +
); } @@ -141,6 +148,10 @@ export function MessageBubble({ inputs={screen.inputs} submitLabel={screen.submit?.label || "Continue"} onSubmit={onFormSubmit} + socialLogins={screen.socialLogins} + onSocialLogin={(provider) => + onFormSubmit({ socialLogin: provider }) + } disabled={busy} /> )} @@ -162,6 +173,26 @@ export function MessageBubble({
)} + {screen.type === "noop_screen" && ( +
+
+

+ {screen.instructionText || + "Complete authentication on your device, then click Continue."} +

+
+
+ +
+
+ )} + {screen.type === "magic_login_link" && (
@@ -194,16 +225,21 @@ export function MessageBubble({ function CollapsedForm({ screen, status, + overrideLabel, }: { screen: LoginState; status: "submitting" | "submitted"; + overrideLabel?: string; }) { - const label = - screen.type === "credential_login_form" && screen.inputs + const label = overrideLabel + ? overrideLabel + : screen.type === "credential_login_form" && screen.inputs ? screen.inputs.map((i) => i.label || i.name).join(", ") : screen.type === "choice_screen" ? "Selection" - : "Link"; + : screen.type === "noop_screen" + ? "Continue" + : "Link"; return (
diff --git a/src/hooks/use-login-session.ts b/src/hooks/use-login-session.ts index 7cde266..196e39e 100644 --- a/src/hooks/use-login-session.ts +++ b/src/hooks/use-login-session.ts @@ -106,6 +106,9 @@ export function useLoginSession() { const [formStatuses, setFormStatuses] = useState>( {}, ); + const [formSubmitLabels, setFormSubmitLabels] = useState< + Record + >({}); const currentFormId = useRef(null); const loadingRetries = useRef(0); @@ -156,7 +159,8 @@ export function useLoginSession() { switch (screen.type) { case "credential_login_form": case "choice_screen": - case "magic_login_link": { + case "magic_login_link": + case "noop_screen": { loadingRetries.current = 0; setCurrentScreen(screen); @@ -380,6 +384,19 @@ export function useLoginSession() { const formId = currentFormId.current; if (formId) { setFormStatuses((prev) => ({ ...prev, [formId]: "submitting" })); + + // Track what was submitted for the collapsed form label + if (values.socialLogin && currentScreen.socialLogins) { + const provider = currentScreen.socialLogins.find( + (sl) => sl.provider === values.socialLogin, + ); + if (provider) { + setFormSubmitLabels((prev) => ({ + ...prev, + [formId]: provider.buttonText, + })); + } + } } setBusy(true); @@ -482,6 +499,7 @@ export function useLoginSession() { setBusy(false); setLogs([]); setFormStatuses({}); + setFormSubmitLabels({}); currentFormId.current = null; targetDomainRef.current = null; loadingRetries.current = 0; @@ -496,6 +514,7 @@ export function useLoginSession() { busy, logs, formStatuses, + formSubmitLabels, activeFormId: currentFormId.current, startSession, submitForm, diff --git a/src/lib/ai-login/agent.ts b/src/lib/ai-login/agent.ts index 0bb4115..d6653d4 100644 --- a/src/lib/ai-login/agent.ts +++ b/src/lib/ai-login/agent.ts @@ -48,6 +48,9 @@ function getScreenLocators(screen: LoginState): string[] { for (const opt of screen.options) locators.push(opt.optionPlaywrightLocator); } + if (screen.socialLogins) { + for (const sl of screen.socialLogins) locators.push(sl.playwrightLocator); + } if (screen.dismissPlaywrightLocator) locators.push(screen.dismissPlaywrightLocator); return locators; @@ -198,6 +201,23 @@ export async function handleScreen( // credential_login_form — fill fields + click submit // ------------------------------------------------------------------ case "credential_login_form": { + // Social login click — user picked a provider instead of filling the form + if (userInput?.socialLogin && screen.socialLogins) { + const provider = screen.socialLogins.find( + (sl) => sl.provider === userInput.socialLogin, + ); + if (provider) { + await clickElement(session.page, provider.playwrightLocator); + return { + nextScreen: null, + message: { + type: "action", + action: `Clicked ${provider.buttonText}`, + }, + }; + } + } + const hasValues = userInput && Object.values(userInput).some((v) => v); if (!hasValues || !screen.inputs || !screen.submit) { return { @@ -303,6 +323,25 @@ export async function handleScreen( }; } + // ------------------------------------------------------------------ + // noop_screen — nothing to do in the browser, user acts externally + // ------------------------------------------------------------------ + case "noop_screen": { + if (!userInput?.continue) { + return { + nextScreen: null, + message: { type: "input_request", screen }, + }; + } + + // No-op — the user already completed the action on their device. + // The route will re-analyze the page via SSE. + return { + nextScreen: null, + message: { type: "action", action: "Continuing after external auth" }, + }; + } + // ------------------------------------------------------------------ // loading_screen — wait and re-analyze // ------------------------------------------------------------------ diff --git a/src/lib/ai-login/prompts.ts b/src/lib/ai-login/prompts.ts index adec9d8..a95e508 100644 --- a/src/lib/ai-login/prompts.ts +++ b/src/lib/ai-login/prompts.ts @@ -18,9 +18,16 @@ Use when you see multiple buttons/links representing choices (account selector, - Most choice screens do NOT have a separate submit button — the options themselves are clickable. Only include the submit field if the page truly requires selecting an option (radio button, select box) AND then clicking a separate "Continue"/"Submit" button to confirm. Do not add submit unless it is clearly required. ### magic_login_link -Use when the page instructs user to check their email for a verification/magic link. +Use when the page instructs user to check their email for a verification/magic link that they need to paste back. - No active input fields - Common patterns: "Check your inbox", "We sent you an email" +- The user must copy a URL and provide it back to the system + +### noop_screen +Use ONLY when there is not primary call to action (CTA) element in the browser — no buttons, no links, no inputs, nothing to click or fill. The user must complete an action entirely outside the browser (e.g. tap a number on their phone, approve a push notification on another device). +- CRITICAL: If the page has clickable buttons (like "Continue", "Verify"), it is NOT a noop_screen — classify it as choice_screen instead +- Extract the instruction text into \`instructionText\` +- Examples: "Tap 58 on your phone to verify" (with no buttons on the page), "Check your device for a notification" (with no buttons on the page) ### blocked_screen Use when a popup/dialog blocks the login flow and must be dismissed. @@ -42,9 +49,15 @@ Priority for robust Playwright locators: 3. Button/link text: button:has-text("Sign In") 4. Data attributes: [data-testid="login-button"] +## Social Login Detection + +When a credential_login_form has social/SSO login buttons (e.g. "Sign in with Google", "Continue with Apple", "Log in with Microsoft"), extract them into the \`socialLogins\` array. Each entry needs: provider name (lowercase: google, apple, microsoft, github, okta, etc.), visible button text, and a Playwright locator. + +If a page has ONLY social login buttons with NO credential inputs, classify it as choice_screen instead. + ## Exclusions -Exclude: hidden/disabled fields, cookie banners, social login buttons (Google/Apple/Microsoft), help/privacy/forgot-password links, sign-up links. +Exclude: hidden/disabled fields, cookie banners, help/privacy/forgot-password links, sign-up links. ## OTP Detection diff --git a/src/lib/ai-login/types.ts b/src/lib/ai-login/types.ts index 6c62c48..4169bb6 100644 --- a/src/lib/ai-login/types.ts +++ b/src/lib/ai-login/types.ts @@ -8,6 +8,7 @@ export const ScreenType = z.enum([ "credential_login_form", "choice_screen", "magic_login_link", + "noop_screen", "logged_in_screen", "loading_screen", "blocked_screen", @@ -55,6 +56,20 @@ export const ChoiceOption = z.object({ .describe("Playwright locator for this option"), }); +export const SocialLoginOption = z.object({ + provider: z + .string() + .describe( + "Provider name in lowercase: google, apple, microsoft, github, okta, etc.", + ), + buttonText: z + .string() + .describe("Visible button text, e.g. 'Continue with Google'"), + playwrightLocator: z + .string() + .describe("Playwright locator for the social login button"), +}); + // --------------------------------------------------------------------------- // Unified login state schema // @@ -83,6 +98,12 @@ export const LoginStateSchema = z.object({ .string() .optional() .describe("Instructions for magic link"), + socialLogins: z + .array(SocialLoginOption) + .optional() + .describe( + "Social/SSO login buttons on the page (alongside credential form)", + ), dismissPlaywrightLocator: z .string() .optional() @@ -92,6 +113,7 @@ export const LoginStateSchema = z.object({ export type LoginState = z.infer; export type InputElementType = z.infer; export type ChoiceOptionType = z.infer; +export type SocialLoginOptionType = z.infer; // --------------------------------------------------------------------------- // Agent messages — returned by handleScreen to describe what happened