Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion src/app/api/chat/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,12 @@ export async function POST(request: NextRequest) {
const hasUserInput = Object.values(values).some((v) => v);
const isUserAction =
hasUserInput &&
["credential_login_form", "choice_screen", "magic_login_link"].includes(
[
"credential_login_form",
"choice_screen",
"magic_login_link",
"noop_screen",
].includes(
screen.type,
);

Expand Down
2 changes: 2 additions & 0 deletions src/components/chat.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ export function Chat() {
busy,
logs,
formStatuses,
formSubmitLabels,
activeFormId,
startSession,
submitForm,
Expand Down Expand Up @@ -157,6 +158,7 @@ export function Chat() {
busy={busy}
activeFormId={activeFormId}
formStatuses={formStatuses}
formSubmitLabels={formSubmitLabels}
showLabel={i === 0 || messages[i - 1].role !== msg.role}
/>
))}
Expand Down
34 changes: 33 additions & 1 deletion src/components/credential-form.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,18 @@ import { Lock } from "lucide-react";
import { Input } from "@/components/ui/input";
import { Button } from "@/components/ui/button";
import { Label } from "@/components/ui/label";
import type { InputElementType } from "@/lib/ai-login/types";
import type {
InputElementType,
SocialLoginOptionType,
} from "@/lib/ai-login/types";
import { cn } from "@/lib/utils";

interface CredentialFormProps {
inputs: InputElementType[];
submitLabel: string;
onSubmit: (values: Record<string, string>) => void;
socialLogins?: SocialLoginOptionType[];
onSocialLogin?: (provider: string) => void;
disabled?: boolean;
}

Expand Down Expand Up @@ -56,6 +61,8 @@ export function CredentialForm({
inputs,
submitLabel,
onSubmit,
socialLogins,
onSocialLogin,
disabled,
}: CredentialFormProps) {
const [values, setValues] = useState<Record<string, string>>({});
Expand Down Expand Up @@ -123,6 +130,31 @@ export function CredentialForm({
{submitLabel}
</Button>
</div>

{/* Social login buttons */}
{socialLogins && socialLogins.length > 0 && onSocialLogin && (
<div className="px-4 pb-4">
<div className="flex items-center gap-3 mb-3">
<div className="h-px flex-1 bg-white/[0.1]" />
<span className="text-[11px] text-white/40">or</span>
<div className="h-px flex-1 bg-white/[0.1]" />
</div>
<div className="space-y-1.5">
{socialLogins.map((sl) => (
<Button
key={sl.provider}
type="button"
variant="outline"
disabled={disabled}
onClick={() => onSocialLogin(sl.provider)}
className="w-full justify-center bg-white/[0.04] border-white/[0.1] text-[13px] text-white/80 hover:bg-white/[0.08] hover:border-white/[0.15] hover:text-white h-auto px-3.5 py-2.5"
>
{sl.buttonText}
</Button>
))}
</div>
</div>
)}
</form>
);
}
44 changes: 40 additions & 4 deletions src/components/message-bubble.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import { Loader2, Check, LockOpen } from "lucide-react";
import type { ChatMessage, FormStatus } from "@/hooks/use-login-session";
import type { LoginState } from "@/lib/ai-login/types";
import { Button } from "@/components/ui/button";
import { CredentialForm } from "./credential-form";
import { ChoiceButtons } from "./choice-buttons";
import { MagicLinkInput } from "./magic-link-input";
Expand All @@ -23,6 +24,7 @@ interface MessageBubbleProps {
busy: boolean;
activeFormId: string | null;
formStatuses: Record<string, FormStatus>;
formSubmitLabels: Record<string, string>;
showLabel?: boolean;
}

Expand All @@ -32,6 +34,7 @@ export function MessageBubble({
busy,
activeFormId,
formStatuses,
formSubmitLabels,
showLabel,
}: MessageBubbleProps) {
// User message
Expand Down Expand Up @@ -125,7 +128,11 @@ export function MessageBubble({
return (
<div>
{assistantLabel}
<CollapsedForm screen={screen} status={status} />
<CollapsedForm
screen={screen}
status={status}
overrideLabel={formSubmitLabels[formId]}
/>
</div>
);
}
Expand All @@ -141,6 +148,10 @@ export function MessageBubble({
inputs={screen.inputs}
submitLabel={screen.submit?.label || "Continue"}
onSubmit={onFormSubmit}
socialLogins={screen.socialLogins}
onSocialLogin={(provider) =>
onFormSubmit({ socialLogin: provider })
}
disabled={busy}
/>
)}
Expand All @@ -162,6 +173,26 @@ export function MessageBubble({
</div>
)}

{screen.type === "noop_screen" && (
<div className="rounded-xl border border-white/[0.12] bg-white/[0.04] overflow-hidden">
<div className="px-4 pt-3 pb-2">
<p className="text-[13px] text-white/80">
{screen.instructionText ||
"Complete authentication on your device, then click Continue."}
</p>
</div>
<div className="px-4 pb-4">
<Button
onClick={() => onFormSubmit({ continue: "true" })}
disabled={busy}
className="w-full bg-white text-[#09090b] text-[13px] font-semibold hover:bg-white/90 active:bg-white/80"
>
Continue
</Button>
</div>
</div>
)}

{screen.type === "magic_login_link" && (
<div className="rounded-xl border border-white/[0.12] bg-white/[0.04] overflow-hidden">
<div className="px-4 pt-3 pb-2">
Expand Down Expand Up @@ -194,16 +225,21 @@ export function MessageBubble({
function CollapsedForm({
screen,
status,
overrideLabel,
}: {
screen: LoginState;
status: "submitting" | "submitted";
overrideLabel?: string;
}) {
const label =
screen.type === "credential_login_form" && screen.inputs
const label = overrideLabel
? overrideLabel
: screen.type === "credential_login_form" && screen.inputs
? screen.inputs.map((i) => i.label || i.name).join(", ")
: screen.type === "choice_screen"
? "Selection"
: "Link";
: screen.type === "noop_screen"
? "Continue"
: "Link";

return (
<div className="flex justify-start w-full">
Expand Down
21 changes: 20 additions & 1 deletion src/hooks/use-login-session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,9 @@ export function useLoginSession() {
const [formStatuses, setFormStatuses] = useState<Record<string, FormStatus>>(
{},
);
const [formSubmitLabels, setFormSubmitLabels] = useState<
Record<string, string>
>({});

const currentFormId = useRef<string | null>(null);
const loadingRetries = useRef(0);
Expand Down Expand Up @@ -156,7 +159,8 @@ export function useLoginSession() {
switch (screen.type) {
case "credential_login_form":
case "choice_screen":
case "magic_login_link": {
case "magic_login_link":
case "noop_screen": {
loadingRetries.current = 0;
setCurrentScreen(screen);

Expand Down Expand Up @@ -380,6 +384,19 @@ export function useLoginSession() {
const formId = currentFormId.current;
if (formId) {
setFormStatuses((prev) => ({ ...prev, [formId]: "submitting" }));

// Track what was submitted for the collapsed form label
if (values.socialLogin && currentScreen.socialLogins) {
const provider = currentScreen.socialLogins.find(
(sl) => sl.provider === values.socialLogin,
);
if (provider) {
setFormSubmitLabels((prev) => ({
...prev,
[formId]: provider.buttonText,
}));
}
}
}

setBusy(true);
Expand Down Expand Up @@ -482,6 +499,7 @@ export function useLoginSession() {
setBusy(false);
setLogs([]);
setFormStatuses({});
setFormSubmitLabels({});
currentFormId.current = null;
targetDomainRef.current = null;
loadingRetries.current = 0;
Expand All @@ -496,6 +514,7 @@ export function useLoginSession() {
busy,
logs,
formStatuses,
formSubmitLabels,
activeFormId: currentFormId.current,
startSession,
submitForm,
Expand Down
39 changes: 39 additions & 0 deletions src/lib/ai-login/agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ function getScreenLocators(screen: LoginState): string[] {
for (const opt of screen.options)
locators.push(opt.optionPlaywrightLocator);
}
if (screen.socialLogins) {
for (const sl of screen.socialLogins) locators.push(sl.playwrightLocator);
}
if (screen.dismissPlaywrightLocator)
locators.push(screen.dismissPlaywrightLocator);
return locators;
Expand Down Expand Up @@ -198,6 +201,23 @@ export async function handleScreen(
// credential_login_form — fill fields + click submit
// ------------------------------------------------------------------
case "credential_login_form": {
// Social login click — user picked a provider instead of filling the form
if (userInput?.socialLogin && screen.socialLogins) {
const provider = screen.socialLogins.find(
(sl) => sl.provider === userInput.socialLogin,
);
if (provider) {
await clickElement(session.page, provider.playwrightLocator);
return {
nextScreen: null,
message: {
type: "action",
action: `Clicked ${provider.buttonText}`,
},
};
}
}

const hasValues = userInput && Object.values(userInput).some((v) => v);
if (!hasValues || !screen.inputs || !screen.submit) {
return {
Expand Down Expand Up @@ -303,6 +323,25 @@ export async function handleScreen(
};
}

// ------------------------------------------------------------------
// noop_screen — nothing to do in the browser, user acts externally
// ------------------------------------------------------------------
case "noop_screen": {
if (!userInput?.continue) {
return {
nextScreen: null,
message: { type: "input_request", screen },
};
}

// No-op — the user already completed the action on their device.
// The route will re-analyze the page via SSE.
return {
nextScreen: null,
message: { type: "action", action: "Continuing after external auth" },
};
}

// ------------------------------------------------------------------
// loading_screen — wait and re-analyze
// ------------------------------------------------------------------
Expand Down
17 changes: 15 additions & 2 deletions src/lib/ai-login/prompts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,16 @@ Use when you see multiple buttons/links representing choices (account selector,
- Most choice screens do NOT have a separate submit button — the options themselves are clickable. Only include the submit field if the page truly requires selecting an option (radio button, select box) AND then clicking a separate "Continue"/"Submit" button to confirm. Do not add submit unless it is clearly required.

### magic_login_link
Use when the page instructs user to check their email for a verification/magic link.
Use when the page instructs user to check their email for a verification/magic link that they need to paste back.
- No active input fields
- Common patterns: "Check your inbox", "We sent you an email"
- The user must copy a URL and provide it back to the system

### noop_screen
Use ONLY when there is not primary call to action (CTA) element in the browser — no buttons, no links, no inputs, nothing to click or fill. The user must complete an action entirely outside the browser (e.g. tap a number on their phone, approve a push notification on another device).
- CRITICAL: If the page has clickable buttons (like "Continue", "Verify"), it is NOT a noop_screen — classify it as choice_screen instead
- Extract the instruction text into \`instructionText\`
- Examples: "Tap 58 on your phone to verify" (with no buttons on the page), "Check your device for a notification" (with no buttons on the page)

### blocked_screen
Use when a popup/dialog blocks the login flow and must be dismissed.
Expand All @@ -42,9 +49,15 @@ Priority for robust Playwright locators:
3. Button/link text: button:has-text("Sign In")
4. Data attributes: [data-testid="login-button"]

## Social Login Detection

When a credential_login_form has social/SSO login buttons (e.g. "Sign in with Google", "Continue with Apple", "Log in with Microsoft"), extract them into the \`socialLogins\` array. Each entry needs: provider name (lowercase: google, apple, microsoft, github, okta, etc.), visible button text, and a Playwright locator.

If a page has ONLY social login buttons with NO credential inputs, classify it as choice_screen instead.

## Exclusions

Exclude: hidden/disabled fields, cookie banners, social login buttons (Google/Apple/Microsoft), help/privacy/forgot-password links, sign-up links.
Exclude: hidden/disabled fields, cookie banners, help/privacy/forgot-password links, sign-up links.

## OTP Detection

Expand Down
22 changes: 22 additions & 0 deletions src/lib/ai-login/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ export const ScreenType = z.enum([
"credential_login_form",
"choice_screen",
"magic_login_link",
"noop_screen",
"logged_in_screen",
"loading_screen",
"blocked_screen",
Expand Down Expand Up @@ -55,6 +56,20 @@ export const ChoiceOption = z.object({
.describe("Playwright locator for this option"),
});

export const SocialLoginOption = z.object({
provider: z
.string()
.describe(
"Provider name in lowercase: google, apple, microsoft, github, okta, etc.",
),
buttonText: z
.string()
.describe("Visible button text, e.g. 'Continue with Google'"),
playwrightLocator: z
.string()
.describe("Playwright locator for the social login button"),
});

// ---------------------------------------------------------------------------
// Unified login state schema
//
Expand Down Expand Up @@ -83,6 +98,12 @@ export const LoginStateSchema = z.object({
.string()
.optional()
.describe("Instructions for magic link"),
socialLogins: z
.array(SocialLoginOption)
.optional()
.describe(
"Social/SSO login buttons on the page (alongside credential form)",
),
dismissPlaywrightLocator: z
.string()
.optional()
Expand All @@ -92,6 +113,7 @@ export const LoginStateSchema = z.object({
export type LoginState = z.infer<typeof LoginStateSchema>;
export type InputElementType = z.infer<typeof InputElement>;
export type ChoiceOptionType = z.infer<typeof ChoiceOption>;
export type SocialLoginOptionType = z.infer<typeof SocialLoginOption>;

// ---------------------------------------------------------------------------
// Agent messages — returned by handleScreen to describe what happened
Expand Down