From a605c4eff517e182accb106be6e2c17cd0cdac89 Mon Sep 17 00:00:00 2001 From: Laxman Reddy Aileni Date: Fri, 11 Sep 2026 00:23:05 +0000 Subject: [PATCH] telemetry: emit auth metrics (userState, loginWithBrowser w/ sessionDuration, didLoadModule login) The plugin did not report the standard toolkit auth telemetry that the VS Code and JetBrains extensions report, so the authentication experience of the Eclipse plugin could not be compared with theirs. Emit the same events: - aws_loginWithBrowser on every login and re-authentication, with the result, the start url, the credential type, whether it renewed an existing connection, and the duration of the previous session for that start url. The latest successful login is recorded in the plugin store so the next login can report how long the session lived for; a first login, and the first login after a sign out, report no duration because there is no previous session to measure. A failed login reports a short reason code, never an error message or a path. - auth_userState with source startup, reporting whether the plugin started connected, expired, or not connected. The state restored from the plugin store is optimistic, so the metric is reported for the first state that follows it, which is the outcome of the re-authentication performed on start up. - toolkit_didLoadModule with module login, reporting that the login view loaded or the reason code of the failure that stopped it from loading. The re-authentication view of this plugin is built from native widgets rather than a web view, so it has no equivalent load step and reports no toolkit_didLoadModule event. The telemetry definitions this plugin generates its metric builders from predated toolkit_didLoadModule and the session duration of aws_loginWithBrowser, so both are added from the shared definitions. An optional reason is added to both metrics as well, matching what the other extensions report on them. The generated builder types session duration as a primitive int, which cannot hold a duration of more than roughly twenty five days in milliseconds and which serializes to zero when it is not set. Sessions are expected to live for months, so the entry is written with the exact millisecond value, and left out entirely when no previous session is known. --- .../definitions/commonDefinitions.json | 39 ++++ .../amazonq/lsp/auth/AuthPluginStore.java | 43 +++++ .../lsp/auth/DefaultAuthStateManager.java | 49 +++++ .../amazonq/lsp/auth/DefaultLoginService.java | 77 +++++++- .../ToolkitLoginWebViewAssetProvider.java | 13 ++ .../telemetry/AuthTelemetryProvider.java | 39 ++++ .../telemetry/AwsTelemetryProvider.java | 66 +++++++ .../telemetry/ToolkitTelemetryProvider.java | 23 +++ .../eclipse/amazonq/util/Constants.java | 2 + .../amazonq/views/LoginViewActionHandler.java | 4 + .../amazonq/lsp/auth/AuthPluginStoreTest.java | 64 +++++++ .../lsp/auth/DefaultAuthStateManagerTest.java | 69 +++++++ .../lsp/auth/DefaultLoginServiceTest.java | 180 +++++++++++++++++- .../ToolkitLoginWebViewAssetProviderTest.java | 75 ++++++++ .../telemetry/AwsTelemetryProviderTest.java | 86 +++++++++ .../ToolkitTelemetryProviderTest.java | 63 ++++++ 16 files changed, 888 insertions(+), 4 deletions(-) create mode 100644 plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/AuthTelemetryProvider.java create mode 100644 plugin/tst/software/aws/toolkits/eclipse/amazonq/providers/assets/ToolkitLoginWebViewAssetProviderTest.java create mode 100644 plugin/tst/software/aws/toolkits/eclipse/amazonq/telemetry/AwsTelemetryProviderTest.java create mode 100644 plugin/tst/software/aws/toolkits/eclipse/amazonq/telemetry/ToolkitTelemetryProviderTest.java diff --git a/plugin/codegen-resources/definitions/commonDefinitions.json b/plugin/codegen-resources/definitions/commonDefinitions.json index 96044d8d1..8cdfe7e90 100644 --- a/plugin/codegen-resources/definitions/commonDefinitions.json +++ b/plugin/codegen-resources/definitions/commonDefinitions.json @@ -2889,9 +2889,17 @@ "type": "reAuthReason", "required": false }, + { + "type": "reason", + "required": false + }, { "type": "result" }, + { + "type": "sessionDuration", + "required": false + }, { "type": "source" }, @@ -7061,6 +7069,37 @@ ], "passive": true }, + { + "name": "toolkit_didLoadModule", + "description": "The module has loaded, i.e it has rendered/resolved/finished. You can use this metric by itself, OR after `toolkit_willOpenModule` + `traceId` to close the loop on an asynchronous operation.", + "metadata": [ + { + "type": "attempts", + "required": false + }, + { + "type": "duration", + "required": false + }, + { + "type": "module", + "required": true + }, + { + "type": "reason", + "required": false + }, + { + "type": "result", + "required": true + }, + { + "type": "version", + "required": false + } + ], + "passive": true + }, { "name": "toolkit_execute", "description": "Emitted whenever a registered command is executed", diff --git a/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/AuthPluginStore.java b/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/AuthPluginStore.java index ff25fc109..7f08a0185 100644 --- a/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/AuthPluginStore.java +++ b/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/AuthPluginStore.java @@ -3,6 +3,9 @@ package software.aws.toolkits.eclipse.amazonq.lsp.auth; +import java.time.Instant; +import java.util.Optional; + import software.aws.toolkits.eclipse.amazonq.configuration.PluginStore; import software.aws.toolkits.eclipse.amazonq.lsp.auth.model.LoginIdcParams; import software.aws.toolkits.eclipse.amazonq.lsp.auth.model.LoginParams; @@ -54,9 +57,49 @@ public String getSsoTokenId() { return pluginStore.get(Constants.SSO_TOKEN_ID); } + /** + * Persists the instant of the latest successful login alongside the start url it was performed + * against. It is used to report how long the previous authentication session lived for. + * + * @param startUrl the start url the login was performed against + * @param loginInstant the instant the login completed + */ + public void setLoginTimestamp(final String startUrl, final Instant loginInstant) { + if (startUrl == null || loginInstant == null) { + return; + } + pluginStore.put(Constants.LOGIN_TIMESTAMP_START_URL_KEY, startUrl); + pluginStore.put(Constants.LOGIN_TIMESTAMP_KEY, String.valueOf(loginInstant.toEpochMilli())); + } + + /** + * Retrieves the instant of the latest successful login for the given start url. An empty value is + * returned when no login has been recorded yet, when the recorded login was performed against a + * different start url, or when the recorded value cannot be parsed. + * + * @param startUrl the start url the login is being performed against + * @return the instant of the previous successful login for that start url + */ + public Optional getLoginTimestamp(final String startUrl) { + String storedStartUrl = pluginStore.get(Constants.LOGIN_TIMESTAMP_START_URL_KEY); + String storedTimestamp = pluginStore.get(Constants.LOGIN_TIMESTAMP_KEY); + + if (startUrl == null || storedStartUrl == null || storedTimestamp == null || !storedStartUrl.equals(startUrl)) { + return Optional.empty(); + } + + try { + return Optional.of(Instant.ofEpochMilli(Long.parseLong(storedTimestamp))); + } catch (NumberFormatException ex) { + return Optional.empty(); + } + } + public void clear() { pluginStore.remove(Constants.LOGIN_TYPE_KEY); pluginStore.remove(Constants.LOGIN_IDC_PARAMS_KEY); + pluginStore.remove(Constants.LOGIN_TIMESTAMP_START_URL_KEY); + pluginStore.remove(Constants.LOGIN_TIMESTAMP_KEY); pluginStore.remove(Constants.SSO_TOKEN_ID); } } diff --git a/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultAuthStateManager.java b/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultAuthStateManager.java index 421b6f1c7..75a13b2db 100644 --- a/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultAuthStateManager.java +++ b/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultAuthStateManager.java @@ -10,7 +10,9 @@ import software.aws.toolkits.eclipse.amazonq.lsp.auth.model.LoginParams; import software.aws.toolkits.eclipse.amazonq.lsp.auth.model.LoginType; import software.aws.toolkits.eclipse.amazonq.plugin.Activator; +import software.aws.toolkits.eclipse.amazonq.telemetry.AuthTelemetryProvider; import software.aws.toolkits.eclipse.amazonq.util.AuthUtil; +import software.aws.toolkits.telemetry.TelemetryDefinitions.AuthStatus; /** * Manages authentication state transitions and persistence in the Amazon Q plugin. @@ -43,6 +45,8 @@ public final class DefaultAuthStateManager implements AuthStateManager { private String issuerUrl; // used in AmazonQLspClientImpl.getConnectionMetadata() private String ssoTokenId; // used in logout's invalidateSsoToken params private AuthState previousAuthState = null; + private boolean isRestoringPersistedAuthState = false; + private boolean hasEmittedStartupAuthState = false; public DefaultAuthStateManager(final PluginStore pluginStore) { this.authPluginStore = new AuthPluginStore(pluginStore); @@ -132,6 +136,43 @@ private void updateState(final AuthStateType authStatusType, final LoginType log } } previousAuthState = newAuthState; + + emitStartupAuthStateMetric(newAuthState); + } + + /** + * Reports the authentication state observed at startup, once per plugin session. + * + * The state restored from the plugin store is optimistic: a stored connection is assumed to still be + * valid until the re-authentication performed on start up resolves it. The optimistic state is + * therefore skipped and the metric is reported for the state that follows it, which is the outcome + * of that re-authentication. A restored logged out state needs no re-authentication and is + * definitive right away. + * + * @param authState the state the plugin transitioned to + * @see #syncAuthStateWithPluginStore() + * @see DefaultLoginService + */ + private void emitStartupAuthStateMetric(final AuthState authState) { + if (hasEmittedStartupAuthState || isRestoringPersistedAuthState) { + return; + } + hasEmittedStartupAuthState = true; + + AuthTelemetryProvider.emitUserStateOnStartupMetric(toAuthStatus(authState.authStateType()), authState.issuerUrl()); + } + + private static AuthStatus toAuthStatus(final AuthStateType authStateType) { + switch (authStateType) { + case LOGGED_IN: + return AuthStatus.CONNECTED; + case EXPIRED: + return AuthStatus.EXPIRED; + case LOGGED_OUT: + return AuthStatus.NOT_CONNECTED; + default: + return AuthStatus.UNKNOWN; + } } private void syncAuthStateWithPluginStore() { @@ -162,10 +203,18 @@ private void syncAuthStateWithPluginStore() { * * @see DefaultLoginService constructor that handles the re-authentication on LoginService start up */ + boolean restoreFailed = false; try { + isRestoringPersistedAuthState = true; toLoggedIn(loginType, loginParams, ssoTokenId); } catch (Exception ex) { Activator.getLogger().error("Failed to transition to a logged in state after syncing auth state with the persistent store", ex); + restoreFailed = true; + } finally { + isRestoringPersistedAuthState = false; + } + + if (restoreFailed) { toLoggedOut(); } } diff --git a/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultLoginService.java b/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultLoginService.java index f2f9df388..225420c90 100644 --- a/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultLoginService.java +++ b/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultLoginService.java @@ -3,8 +3,11 @@ package software.aws.toolkits.eclipse.amazonq.lsp.auth; +import java.time.Duration; +import java.time.Instant; import java.util.Objects; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.atomic.AtomicReference; import software.aws.toolkits.eclipse.amazonq.configuration.PluginStore; @@ -18,7 +21,11 @@ import software.aws.toolkits.eclipse.amazonq.lsp.model.UpdateCredentialsPayload; import software.aws.toolkits.eclipse.amazonq.plugin.Activator; import software.aws.toolkits.eclipse.amazonq.providers.lsp.LspProvider; +import software.aws.toolkits.eclipse.amazonq.telemetry.AwsTelemetryProvider; +import software.aws.toolkits.eclipse.amazonq.telemetry.AwsTelemetryProvider.BrowserLoginParams; import software.aws.toolkits.eclipse.amazonq.util.AuthUtil; +import software.aws.toolkits.telemetry.TelemetryDefinitions.CredentialType; +import software.aws.toolkits.telemetry.TelemetryDefinitions.Result; /** * Core authentication service for the Amazon Q Eclipse plugin that manages @@ -41,11 +48,13 @@ public final class DefaultLoginService implements LoginService { private AuthStateManager authStateManager; private AuthTokenService authTokenService; private AuthCredentialsService authCredentialsService; + private AuthPluginStore authPluginStore; private DefaultLoginService(final Builder builder) { this.authStateManager = Objects.requireNonNull(builder.authStateManager, "authStateManager cannot be null"); this.authTokenService = Objects.requireNonNull(builder.authTokenService, "authTokenService cannot be null"); this.authCredentialsService = Objects.requireNonNull(builder.authCredentialsService, "authCredentialsService cannot be null"); + this.authPluginStore = new AuthPluginStore(Objects.requireNonNull(builder.pluginStore, "pluginStore cannot be null")); if (builder.initializeOnStartUp) { AuthState authState = authStateManager.getAuthState(); @@ -73,7 +82,7 @@ public CompletableFuture login(final LoginType loginType, final LoginParam Activator.getLogger().info("Attempting to login..."); - return processLogin(loginType, loginParams, true) + return processLogin(loginType, loginParams, true, false) .exceptionally(throwable -> { Activator.getLogger().error("Failed to log in", throwable); logout(); @@ -141,7 +150,7 @@ public CompletableFuture reAuthenticate(final boolean loginOnInvalidToken) Activator.getLogger().info("Attempting to re-authenticate..."); - return processLogin(authState.loginType(), authState.loginParams(), loginOnInvalidToken) + return processLogin(authState.loginType(), authState.loginParams(), loginOnInvalidToken, true) .exceptionally(throwable -> { Activator.getLogger().error("Failed to re-authenticate", throwable); logout(); @@ -154,7 +163,8 @@ public AuthState getAuthState() { return authStateManager.getAuthState(); } - CompletableFuture processLogin(final LoginType loginType, final LoginParams loginParams, final boolean loginOnInvalidToken) { + CompletableFuture processLogin(final LoginType loginType, final LoginParams loginParams, final boolean loginOnInvalidToken, + final boolean isReAuth) { AuthUtil.validateLoginParameters(loginType, loginParams); final AtomicReference ssoTokenId = new AtomicReference<>(); // Saved for logout @@ -172,7 +182,24 @@ CompletableFuture processLogin(final LoginType loginType, final LoginParam }) .thenRun(() -> { authStateManager.toLoggedIn(loginType, loginParams, ssoTokenId.get()); + if (loginOnInvalidToken) { + emitBrowserLoginMetric(loginType, loginParams, isReAuth, Result.SUCCEEDED, null); + } Activator.getLogger().info("Successfully logged in"); + }) + /* + * Reports the outcome of the login itself. The steps that follow are not part of the login, + * so they are wired after this stage to keep them out of the metric. + * + * Only logins that were allowed to open the browser are reported. The re-authentication + * performed on start up passes loginOnInvalidToken=false, it refreshes the cached token + * silently and would otherwise report a browser login (and a session duration) on every + * start of the IDE. + */ + .whenComplete((unused, throwable) -> { + if (throwable != null && loginOnInvalidToken) { + emitBrowserLoginMetric(loginType, loginParams, isReAuth, Result.FAILED, getReasonCode(throwable)); + } }).thenRun(() -> { CustomizationUtil.triggerChangeConfigurationNotification(); }).exceptionally(throwable -> { @@ -180,6 +207,50 @@ CompletableFuture processLogin(final LoginType loginType, final LoginParam }); } + /** + * Emits the browser login metric, reporting how long the previous authentication session for the + * same start url lived for. + * + * The session duration is only known once a login has been recorded for that start url, so it is + * left out of the first login and of the first login that follows a sign out, which clears the + * recorded login. A successful login becomes the new reference point for the next one. + * + * @param loginType the type of connection being authenticated + * @param loginParams the parameters of the connection being authenticated + * @param isReAuth whether the login renews an existing connection + * @param result whether the login succeeded + * @param reason a short reason code when the login failed, null otherwise + */ + private void emitBrowserLoginMetric(final LoginType loginType, final LoginParams loginParams, final boolean isReAuth, + final Result result, final String reason) { + String credentialStartUrl = AuthUtil.getIssuerUrl(loginType, loginParams); + + // The start url identifies the authentication session, a metric without it carries no signal. + if (credentialStartUrl == null || credentialStartUrl.isBlank()) { + return; + } + + Long sessionDuration = null; + if (result == Result.SUCCEEDED) { + Instant loginInstant = Instant.now(); + sessionDuration = authPluginStore.getLoginTimestamp(credentialStartUrl) + .map(previousLogin -> Duration.between(previousLogin, loginInstant).toMillis()) + .filter(duration -> duration >= 0) // guards against a recorded login dated in the future + .orElse(null); + authPluginStore.setLoginTimestamp(credentialStartUrl, loginInstant); + } + + AwsTelemetryProvider.emitLoginWithBrowserEvent(new BrowserLoginParams(credentialStartUrl, + CredentialType.BEARER_TOKEN, isReAuth, result, reason, sessionDuration)); + } + + private static String getReasonCode(final Throwable throwable) { + Throwable cause = throwable instanceof CompletionException && throwable.getCause() != null + ? throwable.getCause() + : throwable; + return cause.getClass().getSimpleName(); + } + public static class Builder { private LspProvider lspProvider; private PluginStore pluginStore; diff --git a/plugin/src/software/aws/toolkits/eclipse/amazonq/providers/assets/ToolkitLoginWebViewAssetProvider.java b/plugin/src/software/aws/toolkits/eclipse/amazonq/providers/assets/ToolkitLoginWebViewAssetProvider.java index 343d6ec79..372306395 100644 --- a/plugin/src/software/aws/toolkits/eclipse/amazonq/providers/assets/ToolkitLoginWebViewAssetProvider.java +++ b/plugin/src/software/aws/toolkits/eclipse/amazonq/providers/assets/ToolkitLoginWebViewAssetProvider.java @@ -16,6 +16,7 @@ import software.aws.toolkits.eclipse.amazonq.broker.events.ToolkitLoginWebViewAssetState; import software.aws.toolkits.eclipse.amazonq.plugin.Activator; +import software.aws.toolkits.eclipse.amazonq.telemetry.ToolkitTelemetryProvider; import software.aws.toolkits.eclipse.amazonq.telemetry.UiTelemetryProvider; import software.aws.toolkits.eclipse.amazonq.util.PluginUtils; import software.aws.toolkits.eclipse.amazonq.util.ThemeDetector; @@ -26,9 +27,13 @@ import software.aws.toolkits.eclipse.amazonq.views.ViewActionHandler; import software.aws.toolkits.eclipse.amazonq.views.ViewCommandParser; import software.aws.toolkits.eclipse.amazonq.views.ViewConstants; +import software.aws.toolkits.telemetry.TelemetryDefinitions.Result; public final class ToolkitLoginWebViewAssetProvider extends WebViewAssetProvider { + private static final String DEPENDENCY_MISSING_REASON = "DependencyMissing"; + private static final String ASSET_LOAD_FAILED_REASON = "AssetLoadFailed"; + private WebviewAssetServer webviewAssetServer; private static final ThemeDetector THEME_DETECTOR = new ThemeDetector(); private final ViewCommandParser commandParser; @@ -47,6 +52,10 @@ public void initialize() { if (content.isEmpty()) { ThreadingUtils.executeAsyncTask(() -> { content = resolveContent(); + if (content.isEmpty()) { + ToolkitTelemetryProvider.emitDidLoadModuleEventMetric(ToolkitTelemetryProvider.LOGIN_MODULE, + Result.FAILED, DEPENDENCY_MISSING_REASON); + } Activator.getEventBroker().post(ToolkitLoginWebViewAssetState.class, content.isPresent() ? ToolkitLoginWebViewAssetState.RESOLVED : ToolkitLoginWebViewAssetState.DEPENDENCY_MISSING); @@ -93,6 +102,8 @@ private Optional resolveContent() { webviewAssetServer = new WebviewAssetServer(); var result = webviewAssetServer.resolve(jsDirectoryPath); if (!result) { + ToolkitTelemetryProvider.emitDidLoadModuleEventMetric(ToolkitTelemetryProvider.LOGIN_MODULE, + Result.FAILED, ASSET_LOAD_FAILED_REASON); return Optional.of("Failed to load JS"); } var loginJsPath = webviewAssetServer.getUri() + "getStart.js"; @@ -146,6 +157,8 @@ private Optional resolveContent() { """, loginJsPath, loginJsPath, loginJsPath, getWaitFunction(), isDarkTheme)); } catch (IOException e) { + ToolkitTelemetryProvider.emitDidLoadModuleEventMetric(ToolkitTelemetryProvider.LOGIN_MODULE, + Result.FAILED, DEPENDENCY_MISSING_REASON); return Optional.of("Failed to load JS"); } } diff --git a/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/AuthTelemetryProvider.java b/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/AuthTelemetryProvider.java new file mode 100644 index 000000000..82b8c0f9a --- /dev/null +++ b/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/AuthTelemetryProvider.java @@ -0,0 +1,39 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.aws.toolkits.eclipse.amazonq.telemetry; + +import java.time.Instant; + +import software.amazon.awssdk.services.toolkittelemetry.model.MetricDatum; +import software.aws.toolkits.eclipse.amazonq.plugin.Activator; +import software.aws.toolkits.telemetry.AuthTelemetry; +import software.aws.toolkits.telemetry.TelemetryDefinitions.AuthStatus; + +public final class AuthTelemetryProvider { + + private static final String STARTUP_SOURCE = "startup"; + + private AuthTelemetryProvider() { + //prevent instantiation + } + + /** + * Reports the authentication state observed when the plugin starts. + * + * @param authStatus the authentication state the plugin started in + * @param credentialStartUrl the start url of the restored connection, null when there is none + */ + public static void emitUserStateOnStartupMetric(final AuthStatus authStatus, final String credentialStartUrl) { + MetricDatum metricDatum = AuthTelemetry.UserStateEvent() + .authStatus(authStatus) + .credentialStartUrl(credentialStartUrl) + .source(STARTUP_SOURCE) + .passive(true) + .createTime(Instant.now()) + .value(1.0) + .build(); + Activator.getTelemetryService().emitMetric(metricDatum); + } + +} diff --git a/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/AwsTelemetryProvider.java b/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/AwsTelemetryProvider.java index 89d382fed..99679ed1f 100644 --- a/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/AwsTelemetryProvider.java +++ b/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/AwsTelemetryProvider.java @@ -4,13 +4,22 @@ package software.aws.toolkits.eclipse.amazonq.telemetry; import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import software.amazon.awssdk.services.toolkittelemetry.model.MetadataEntry; import software.amazon.awssdk.services.toolkittelemetry.model.MetricDatum; import software.aws.toolkits.eclipse.amazonq.plugin.Activator; import software.aws.toolkits.telemetry.AwsTelemetry; +import software.aws.toolkits.telemetry.TelemetryDefinitions.CredentialType; +import software.aws.toolkits.telemetry.TelemetryDefinitions.Result; public final class AwsTelemetryProvider { + private static final String SESSION_DURATION_KEY = "sessionDuration"; + private static final String AUTH_VIEW_SOURCE = "authView"; + private static final String RE_AUTH_SOURCE = "reAuth"; + private AwsTelemetryProvider() { //prevent instantiation } @@ -26,4 +35,61 @@ public static void emitModifySettingEvent(final String settingId, final String s Activator.getTelemetryService().emitMetric(metricDatum); } + public static void emitLoginWithBrowserEvent(final BrowserLoginParams params) { + MetricDatum metricDatum = AwsTelemetry.LoginWithBrowserEvent() + .credentialStartUrl(params.credentialStartUrl()) + .credentialType(params.credentialType()) + .isReAuth(params.isReAuth()) + .result(params.result()) + .reason(params.reason()) + .source(params.isReAuth() ? RE_AUTH_SOURCE : AUTH_VIEW_SOURCE) + .passive(false) + .createTime(Instant.now()) + .value(1.0) + .build(); + Activator.getTelemetryService().emitMetric(withSessionDuration(metricDatum, params.sessionDuration())); + } + + /** + * Rewrites the session duration metadata entry of the given metric. + * + * The generated builder types session duration as a primitive int, which serializes to zero when + * it is not set and cannot hold durations longer than roughly twenty five days in milliseconds. + * Authentication sessions are expected to live for months, and a first login has no previous + * session at all, so the entry is written with the exact millisecond value when a previous session + * is known and removed when it is not. + * + * @param metricDatum the metric built by the generated builder + * @param sessionDuration milliseconds since the previous successful login, or null when unknown + * @return the metric carrying an accurate session duration, or none at all + */ + private static MetricDatum withSessionDuration(final MetricDatum metricDatum, final Long sessionDuration) { + List metadata = new ArrayList<>(); + for (MetadataEntry entry : metricDatum.metadata()) { + if (!SESSION_DURATION_KEY.equals(entry.key())) { + metadata.add(entry); + } + } + if (sessionDuration != null) { + metadata.add(MetadataEntry.builder() + .key(SESSION_DURATION_KEY) + .value(String.valueOf(sessionDuration)) + .build()); + } + return metricDatum.toBuilder().metadata(metadata).build(); + } + + /** + * Parameters of the browser login metric. + * + * @param credentialStartUrl the start url the login was performed against + * @param credentialType the type of credentials the login produces + * @param isReAuth whether the login renewed an existing connection + * @param result whether the login succeeded + * @param reason a short reason code when the login failed, null otherwise + * @param sessionDuration milliseconds since the previous successful login, null when unknown + */ + public record BrowserLoginParams(String credentialStartUrl, CredentialType credentialType, boolean isReAuth, + Result result, String reason, Long sessionDuration) { }; + } diff --git a/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/ToolkitTelemetryProvider.java b/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/ToolkitTelemetryProvider.java index 6724a6311..fa4aa1042 100644 --- a/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/ToolkitTelemetryProvider.java +++ b/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/ToolkitTelemetryProvider.java @@ -11,6 +11,8 @@ import java.util.Set; public final class ToolkitTelemetryProvider { + public static final String LOGIN_MODULE = "login"; + private static final Set NON_PASSIVE = Set.of("ellipsesMenu", "statusBar", "shortcut"); private ToolkitTelemetryProvider() { @@ -59,6 +61,27 @@ public static void emitCloseModuleEventMetric(final String module, final String .build(); Activator.getTelemetryService().emitMetric(metadata); } + + /** + * Reports that a module finished loading, or failed to load. The module names are shared with the + * other Amazon Q IDE plugins, so pass one of the constants declared on this class. + * + * @param module the module that finished loading + * @param result whether the module loaded + * @param reason a short reason code when the module failed to load, null otherwise + */ + public static void emitDidLoadModuleEventMetric(final String module, final Result result, final String reason) { + MetricDatum metadata = ToolkitTelemetry.DidLoadModuleEvent() + .module(module) + .result(result) + .reason(reason) + .passive(true) + .createTime(Instant.now()) + .value(1.0) + .build(); + Activator.getTelemetryService().emitMetric(metadata); + } + private static String mapModuleId(final String viewId) { String page = viewId.substring(viewId.lastIndexOf(".") + 1); switch (page) { diff --git a/plugin/src/software/aws/toolkits/eclipse/amazonq/util/Constants.java b/plugin/src/software/aws/toolkits/eclipse/amazonq/util/Constants.java index 82294da69..da26120c8 100644 --- a/plugin/src/software/aws/toolkits/eclipse/amazonq/util/Constants.java +++ b/plugin/src/software/aws/toolkits/eclipse/amazonq/util/Constants.java @@ -32,6 +32,8 @@ private Constants() { public static final String DEFAULT_Q_FOUNDATION_DISPLAY_NAME = "Amazon Q foundation (Default)"; public static final String LOGIN_TYPE_KEY = "LOGIN_TYPE"; public static final String LOGIN_IDC_PARAMS_KEY = "IDC_PARAMS"; + public static final String LOGIN_TIMESTAMP_KEY = "LOGIN_TIMESTAMP"; + public static final String LOGIN_TIMESTAMP_START_URL_KEY = "LOGIN_TIMESTAMP_START_URL"; public static final String SSO_TOKEN_ID = "SSO_TOKEN_IN"; public static final String AWS_BUILDER_ID_URL = "https://view.awsapps.com/start"; public static final String IDC_PROFILE_NAME = "eclipse-q-profile"; diff --git a/plugin/src/software/aws/toolkits/eclipse/amazonq/views/LoginViewActionHandler.java b/plugin/src/software/aws/toolkits/eclipse/amazonq/views/LoginViewActionHandler.java index 0581914ef..c0d1f91b1 100644 --- a/plugin/src/software/aws/toolkits/eclipse/amazonq/views/LoginViewActionHandler.java +++ b/plugin/src/software/aws/toolkits/eclipse/amazonq/views/LoginViewActionHandler.java @@ -20,6 +20,7 @@ import software.aws.toolkits.eclipse.amazonq.lsp.auth.model.LoginParams; import software.aws.toolkits.eclipse.amazonq.lsp.auth.model.LoginType; import software.aws.toolkits.eclipse.amazonq.plugin.Activator; +import software.aws.toolkits.eclipse.amazonq.telemetry.ToolkitTelemetryProvider; import software.aws.toolkits.eclipse.amazonq.util.AwsRegion; import software.aws.toolkits.eclipse.amazonq.util.JsonHandler; import software.aws.toolkits.eclipse.amazonq.util.PluginUtils; @@ -28,6 +29,7 @@ import software.aws.toolkits.eclipse.amazonq.views.model.Command; import software.aws.toolkits.eclipse.amazonq.views.model.ParsedCommand; import software.aws.toolkits.eclipse.amazonq.views.model.QDeveloperProfile; +import software.aws.toolkits.telemetry.TelemetryDefinitions.Result; public class LoginViewActionHandler implements ViewActionHandler { @@ -118,6 +120,8 @@ public final void handleCommand(final ParsedCommand parsedCommand, final Browser browser.execute("changeTheme(" + THEME_DETECTOR.isDarkTheme() + ");"); browser.execute(String.format("ideClient.prepareUi(%s)", js)); browser.execute("ideClient.updateAuthorization('')"); + // The login webview reports onLoad once its assets have finished loading. + ToolkitTelemetryProvider.emitDidLoadModuleEventMetric(ToolkitTelemetryProvider.LOGIN_MODULE, Result.SUCCEEDED, null); break; case ON_SELECT_PROFILE: QDeveloperProfile developerProfile = JSON_HANDLER.convertObject(params, QDeveloperProfile.class); diff --git a/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/AuthPluginStoreTest.java b/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/AuthPluginStoreTest.java index 648f1efb8..f5a84fa6f 100644 --- a/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/AuthPluginStoreTest.java +++ b/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/AuthPluginStoreTest.java @@ -5,9 +5,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; +import java.time.Instant; +import java.util.Optional; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; @@ -116,12 +121,71 @@ public void testGetSsoTokenId() { assertEquals(expectedToken, result); } + @Test + public void testSetLoginTimestamp() { + Instant loginInstant = Instant.ofEpochMilli(1700000000000L); + + authPluginStore.setLoginTimestamp("https://example.com", loginInstant); + + verify(pluginStore).put(Constants.LOGIN_TIMESTAMP_START_URL_KEY, "https://example.com"); + verify(pluginStore).put(Constants.LOGIN_TIMESTAMP_KEY, "1700000000000"); + } + + @Test + public void testSetLoginTimestampWithNullStartUrlDoesNotStore() { + authPluginStore.setLoginTimestamp(null, Instant.ofEpochMilli(1700000000000L)); + + verifyNoInteractions(pluginStore); + } + + @Test + public void testGetLoginTimestampForSameStartUrl() { + when(pluginStore.get(Constants.LOGIN_TIMESTAMP_START_URL_KEY)).thenReturn("https://example.com"); + when(pluginStore.get(Constants.LOGIN_TIMESTAMP_KEY)).thenReturn("1700000000000"); + + Optional result = authPluginStore.getLoginTimestamp("https://example.com"); + + assertEquals(Optional.of(Instant.ofEpochMilli(1700000000000L)), result); + } + + @Test + public void testGetLoginTimestampForDifferentStartUrlReturnsEmpty() { + when(pluginStore.get(Constants.LOGIN_TIMESTAMP_START_URL_KEY)).thenReturn("https://example.com"); + when(pluginStore.get(Constants.LOGIN_TIMESTAMP_KEY)).thenReturn("1700000000000"); + + Optional result = authPluginStore.getLoginTimestamp("https://other.example.com"); + + assertTrue(result.isEmpty()); + } + + @Test + public void testGetLoginTimestampWhenNoTimestampStoredReturnsEmpty() { + when(pluginStore.get(Constants.LOGIN_TIMESTAMP_START_URL_KEY)).thenReturn(null); + when(pluginStore.get(Constants.LOGIN_TIMESTAMP_KEY)).thenReturn(null); + + Optional result = authPluginStore.getLoginTimestamp("https://example.com"); + + assertTrue(result.isEmpty()); + } + + @Test + public void testGetLoginTimestampWhenTimestampNotANumberReturnsEmpty() { + when(pluginStore.get(Constants.LOGIN_TIMESTAMP_START_URL_KEY)).thenReturn("https://example.com"); + when(pluginStore.get(Constants.LOGIN_TIMESTAMP_KEY)).thenReturn("not-a-timestamp"); + + Optional result = authPluginStore.getLoginTimestamp("https://example.com"); + + assertTrue(result.isEmpty()); + } + @Test public void testClear() { authPluginStore.clear(); verify(pluginStore).remove(Constants.LOGIN_TYPE_KEY); verify(pluginStore).remove(Constants.LOGIN_IDC_PARAMS_KEY); + verify(pluginStore).remove(Constants.LOGIN_TIMESTAMP_START_URL_KEY); + verify(pluginStore).remove(Constants.LOGIN_TIMESTAMP_KEY); verify(pluginStore).remove(Constants.SSO_TOKEN_ID); } } diff --git a/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultAuthStateManagerTest.java b/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultAuthStateManagerTest.java index ee94f1315..43f357252 100644 --- a/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultAuthStateManagerTest.java +++ b/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultAuthStateManagerTest.java @@ -7,7 +7,10 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -18,6 +21,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.MockitoAnnotations; import software.aws.toolkits.eclipse.amazonq.configuration.PluginStore; @@ -28,7 +32,9 @@ import software.aws.toolkits.eclipse.amazonq.lsp.auth.model.LoginIdcParams; import software.aws.toolkits.eclipse.amazonq.lsp.auth.model.LoginParams; import software.aws.toolkits.eclipse.amazonq.lsp.auth.model.LoginType; +import software.aws.toolkits.eclipse.amazonq.telemetry.AuthTelemetryProvider; import software.aws.toolkits.eclipse.amazonq.util.Constants; +import software.aws.toolkits.telemetry.TelemetryDefinitions.AuthStatus; class DefaultAuthStateManagerTest { @@ -221,6 +227,69 @@ void syncAuthStateWithPluginStoreWithNoStoredCredentialsSetsLoggedOut() { assertNull(state.issuerUrl()); } + @Test + void syncAuthStateWithPluginStoreWithNoStoredCredentialsEmitsNotConnectedStartupUserState() { + when(pluginStore.get(Constants.LOGIN_TYPE_KEY)).thenReturn(LoginType.NONE.name()); + + try (MockedStatic mockedAuthTelemetryProvider = mockStatic(AuthTelemetryProvider.class)) { + new DefaultAuthStateManager(pluginStore); + + mockedAuthTelemetryProvider + .verify(() -> AuthTelemetryProvider.emitUserStateOnStartupMetric(AuthStatus.NOT_CONNECTED, null)); + } + } + + @Test + void syncAuthStateWithPluginStoreWithStoredCredentialsDefersStartupUserStateUntilReAuthenticationResolves() { + when(pluginStore.get(Constants.LOGIN_TYPE_KEY)).thenReturn(LoginType.BUILDER_ID.name()); + when(pluginStore.getObject(Constants.LOGIN_IDC_PARAMS_KEY, LoginIdcParams.class)) + .thenReturn(loginParams.getLoginIdcParams()); + when(pluginStore.get(Constants.SSO_TOKEN_ID)).thenReturn("ssoTokenId"); + + try (MockedStatic mockedAuthTelemetryProvider = mockStatic(AuthTelemetryProvider.class)) { + DefaultAuthStateManager newManager = new DefaultAuthStateManager(pluginStore); + + mockedAuthTelemetryProvider.verifyNoInteractions(); + + newManager.toLoggedIn(LoginType.BUILDER_ID, loginParams, "ssoTokenId"); + + mockedAuthTelemetryProvider.verify(() -> AuthTelemetryProvider + .emitUserStateOnStartupMetric(AuthStatus.CONNECTED, Constants.AWS_BUILDER_ID_URL)); + } + } + + @Test + void startupUserStateReportsExpiredWhenRestoredSessionExpires() { + when(pluginStore.get(Constants.LOGIN_TYPE_KEY)).thenReturn(LoginType.BUILDER_ID.name()); + when(pluginStore.getObject(Constants.LOGIN_IDC_PARAMS_KEY, LoginIdcParams.class)) + .thenReturn(loginParams.getLoginIdcParams()); + when(pluginStore.get(Constants.SSO_TOKEN_ID)).thenReturn("ssoTokenId"); + + try (MockedStatic mockedAuthTelemetryProvider = mockStatic(AuthTelemetryProvider.class)) { + DefaultAuthStateManager newManager = new DefaultAuthStateManager(pluginStore); + + newManager.toExpired(); + + mockedAuthTelemetryProvider.verify(() -> AuthTelemetryProvider + .emitUserStateOnStartupMetric(AuthStatus.EXPIRED, Constants.AWS_BUILDER_ID_URL)); + } + } + + @Test + void startupUserStateIsEmittedOnlyOncePerSession() { + when(pluginStore.get(Constants.LOGIN_TYPE_KEY)).thenReturn(LoginType.NONE.name()); + + try (MockedStatic mockedAuthTelemetryProvider = mockStatic(AuthTelemetryProvider.class)) { + DefaultAuthStateManager newManager = new DefaultAuthStateManager(pluginStore); + + newManager.toLoggedIn(LoginType.BUILDER_ID, loginParams, "ssoTokenId"); + newManager.toExpired(); + + mockedAuthTelemetryProvider.verify( + () -> AuthTelemetryProvider.emitUserStateOnStartupMetric(any(AuthStatus.class), any()), times(1)); + } + } + @Test void setAuthStateFieldsSuccess() { String ssoTokenId = "ssoTokenId"; diff --git a/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultLoginServiceTest.java b/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultLoginServiceTest.java index b2a95ddd1..4672b74de 100644 --- a/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultLoginServiceTest.java +++ b/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultLoginServiceTest.java @@ -3,23 +3,32 @@ package software.aws.toolkits.eclipse.amazonq.lsp.auth; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import java.time.Duration; +import java.time.Instant; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; import software.aws.toolkits.eclipse.amazonq.configuration.DefaultPluginStore; @@ -37,9 +46,13 @@ import software.aws.toolkits.eclipse.amazonq.lsp.model.UpdateCredentialsPayload; import software.aws.toolkits.eclipse.amazonq.plugin.Activator; import software.aws.toolkits.eclipse.amazonq.providers.lsp.LspProvider; +import software.aws.toolkits.eclipse.amazonq.telemetry.AwsTelemetryProvider; +import software.aws.toolkits.eclipse.amazonq.telemetry.AwsTelemetryProvider.BrowserLoginParams; import software.aws.toolkits.eclipse.amazonq.util.AuthUtil; import software.aws.toolkits.eclipse.amazonq.util.Constants; import software.aws.toolkits.eclipse.amazonq.util.LoggingService; +import software.aws.toolkits.telemetry.TelemetryDefinitions.CredentialType; +import software.aws.toolkits.telemetry.TelemetryDefinitions.Result; public final class DefaultLoginServiceTest { @@ -57,6 +70,7 @@ public final class DefaultLoginServiceTest { private static GetSsoTokenResult expectedSsoToken; private static SsoToken ssoToken; private static MockedStatic mockedCustomizationUtil; + private static MockedStatic mockedAwsTelemetryProvider; @BeforeEach public void setUp() { @@ -72,6 +86,7 @@ public void setUp() { mockedAuthUtil = mockStatic(AuthUtil.class); mockedActivator.when(Activator::getLspProvider).thenReturn(mockLspProvider); mockedCustomizationUtil = mockStatic(CustomizationUtil.class); + mockedAwsTelemetryProvider = mockStatic(AwsTelemetryProvider.class); updateCredentialsPayload = mock(UpdateCredentialsPayload.class); when(updateCredentialsPayload.data()).thenReturn("data"); @@ -99,6 +114,7 @@ void tearDown() throws Exception { mockedActivator.close(); mockedAuthUtil.close(); mockedCustomizationUtil.close(); + mockedAwsTelemetryProvider.close(); } @Test @@ -417,6 +433,163 @@ void processLoginIdcWithLoginOnInvalidTokenSuccess() throws Exception { verify(mockLoggingService).info("Successfully logged in"); } + @Test + void processLoginEmitsBrowserLoginSucceededWithoutSessionDurationOnFirstLogin() throws Exception { + LoginType loginType = LoginType.BUILDER_ID; + LoginParams loginParams = createLoginParams(createLoginIdcParams("test-region", "test-url")); + mockedAuthUtil.when(() -> AuthUtil.getIssuerUrl(loginType, loginParams)).thenReturn(Constants.AWS_BUILDER_ID_URL); + + when(mockedAuthTokenService.getSsoToken(loginType, loginParams, true)) + .thenReturn(CompletableFuture.completedFuture(expectedSsoToken)); + when(mockedAuthCredentialsService.updateTokenCredentials(expectedSsoToken.updateCredentialsParams())) + .thenReturn(CompletableFuture.completedFuture(null)); + + invokeProcessLogin(loginType, loginParams, true); + + BrowserLoginParams params = captureBrowserLoginParams(); + assertEquals(Constants.AWS_BUILDER_ID_URL, params.credentialStartUrl()); + assertEquals(CredentialType.BEARER_TOKEN, params.credentialType()); + assertEquals(Result.SUCCEEDED, params.result()); + assertFalse(params.isReAuth()); + assertNull(params.reason()); + assertNull(params.sessionDuration()); + } + + @Test + void processLoginRecordsLoginTimestampOnSuccess() throws Exception { + LoginType loginType = LoginType.BUILDER_ID; + LoginParams loginParams = createLoginParams(createLoginIdcParams("test-region", "test-url")); + mockedAuthUtil.when(() -> AuthUtil.getIssuerUrl(loginType, loginParams)).thenReturn(Constants.AWS_BUILDER_ID_URL); + + when(mockedAuthTokenService.getSsoToken(loginType, loginParams, true)) + .thenReturn(CompletableFuture.completedFuture(expectedSsoToken)); + when(mockedAuthCredentialsService.updateTokenCredentials(expectedSsoToken.updateCredentialsParams())) + .thenReturn(CompletableFuture.completedFuture(null)); + + invokeProcessLogin(loginType, loginParams, true); + + verify(mockPluginStore).put(Constants.LOGIN_TIMESTAMP_START_URL_KEY, Constants.AWS_BUILDER_ID_URL); + verify(mockPluginStore).put(eq(Constants.LOGIN_TIMESTAMP_KEY), any(String.class)); + } + + @Test + void processLoginEmitsBrowserLoginSucceededWithSessionDurationWhenPreviousLoginRecorded() throws Exception { + LoginType loginType = LoginType.BUILDER_ID; + LoginParams loginParams = createLoginParams(createLoginIdcParams("test-region", "test-url")); + mockedAuthUtil.when(() -> AuthUtil.getIssuerUrl(loginType, loginParams)).thenReturn(Constants.AWS_BUILDER_ID_URL); + long previousLoginMillis = Instant.now().minus(Duration.ofDays(30)).toEpochMilli(); + when(mockPluginStore.get(Constants.LOGIN_TIMESTAMP_START_URL_KEY)).thenReturn(Constants.AWS_BUILDER_ID_URL); + when(mockPluginStore.get(Constants.LOGIN_TIMESTAMP_KEY)).thenReturn(String.valueOf(previousLoginMillis)); + + when(mockedAuthTokenService.getSsoToken(loginType, loginParams, true)) + .thenReturn(CompletableFuture.completedFuture(expectedSsoToken)); + when(mockedAuthCredentialsService.updateTokenCredentials(expectedSsoToken.updateCredentialsParams())) + .thenReturn(CompletableFuture.completedFuture(null)); + + invokeProcessLogin(loginType, loginParams, true); + + BrowserLoginParams params = captureBrowserLoginParams(); + assertEquals(Result.SUCCEEDED, params.result()); + assertNotNull(params.sessionDuration()); + assertTrue(params.sessionDuration() >= Duration.ofDays(30).toMillis(), + "session duration should cover the recorded previous login"); + } + + @Test + void processLoginEmitsBrowserLoginWithIsReAuthWhenReAuthenticating() throws Exception { + LoginType loginType = LoginType.BUILDER_ID; + LoginParams loginParams = createLoginParams(createLoginIdcParams("test-region", "test-url")); + mockedAuthUtil.when(() -> AuthUtil.getIssuerUrl(loginType, loginParams)).thenReturn(Constants.AWS_BUILDER_ID_URL); + + when(mockedAuthTokenService.getSsoToken(loginType, loginParams, true)) + .thenReturn(CompletableFuture.completedFuture(expectedSsoToken)); + when(mockedAuthCredentialsService.updateTokenCredentials(expectedSsoToken.updateCredentialsParams())) + .thenReturn(CompletableFuture.completedFuture(null)); + + invokeProcessLogin(loginType, loginParams, true, true); + + BrowserLoginParams params = captureBrowserLoginParams(); + assertTrue(params.isReAuth()); + assertEquals(Result.SUCCEEDED, params.result()); + } + + @Test + void processLoginDoesNotEmitBrowserLoginOnSilentTokenRefresh() throws Exception { + // The re-authentication performed on start up passes loginOnInvalidToken=false: the cached token + // is refreshed without opening the browser, so it is neither a browser login nor a new session. + LoginType loginType = LoginType.BUILDER_ID; + LoginParams loginParams = createLoginParams(createLoginIdcParams("test-region", "test-url")); + mockedAuthUtil.when(() -> AuthUtil.getIssuerUrl(loginType, loginParams)).thenReturn(Constants.AWS_BUILDER_ID_URL); + + when(mockedAuthTokenService.getSsoToken(loginType, loginParams, false)) + .thenReturn(CompletableFuture.completedFuture(expectedSsoToken)); + when(mockedAuthCredentialsService.updateTokenCredentials(expectedSsoToken.updateCredentialsParams())) + .thenReturn(CompletableFuture.completedFuture(null)); + + invokeProcessLogin(loginType, loginParams, false, true); + + mockedAwsTelemetryProvider.verifyNoInteractions(); + verify(mockPluginStore, never()).put(eq(Constants.LOGIN_TIMESTAMP_KEY), any(String.class)); + } + + @Test + void processLoginDoesNotEmitBrowserLoginFailedOnSilentTokenRefresh() { + LoginType loginType = LoginType.BUILDER_ID; + LoginParams loginParams = createLoginParams(createLoginIdcParams("test-region", "test-url")); + mockedAuthUtil.when(() -> AuthUtil.getIssuerUrl(loginType, loginParams)).thenReturn(Constants.AWS_BUILDER_ID_URL); + + when(mockedAuthTokenService.getSsoToken(loginType, loginParams, false)) + .thenReturn(CompletableFuture.failedFuture(new IllegalStateException("token unavailable"))); + + CompletableFuture result = loginService.processLogin(loginType, loginParams, false, true); + + assertThrows(ExecutionException.class, result::get); + mockedAwsTelemetryProvider.verifyNoInteractions(); + } + + @Test + void processLoginEmitsBrowserLoginFailedWithExceptionReason() { + LoginType loginType = LoginType.BUILDER_ID; + LoginParams loginParams = createLoginParams(createLoginIdcParams("test-region", "test-url")); + mockedAuthUtil.when(() -> AuthUtil.getIssuerUrl(loginType, loginParams)).thenReturn(Constants.AWS_BUILDER_ID_URL); + + when(mockedAuthTokenService.getSsoToken(loginType, loginParams, true)) + .thenReturn(CompletableFuture.failedFuture(new IllegalStateException("token unavailable"))); + + CompletableFuture result = loginService.processLogin(loginType, loginParams, true, false); + + assertThrows(ExecutionException.class, result::get); + + BrowserLoginParams params = captureBrowserLoginParams(); + assertEquals(Constants.AWS_BUILDER_ID_URL, params.credentialStartUrl()); + assertEquals(Result.FAILED, params.result()); + assertEquals("IllegalStateException", params.reason()); + assertNull(params.sessionDuration()); + verify(mockPluginStore, never()).put(eq(Constants.LOGIN_TIMESTAMP_KEY), any(String.class)); + } + + @Test + void processLoginDoesNotEmitBrowserLoginWhenStartUrlIsUnknown() throws Exception { + LoginType loginType = LoginType.BUILDER_ID; + LoginParams loginParams = createLoginParams(createLoginIdcParams("test-region", "test-url")); + mockedAuthUtil.when(() -> AuthUtil.getIssuerUrl(loginType, loginParams)).thenReturn(null); + + when(mockedAuthTokenService.getSsoToken(loginType, loginParams, true)) + .thenReturn(CompletableFuture.completedFuture(expectedSsoToken)); + when(mockedAuthCredentialsService.updateTokenCredentials(expectedSsoToken.updateCredentialsParams())) + .thenReturn(CompletableFuture.completedFuture(null)); + + invokeProcessLogin(loginType, loginParams, true); + + mockedAwsTelemetryProvider.verifyNoInteractions(); + } + + private BrowserLoginParams captureBrowserLoginParams() { + ArgumentCaptor captor = ArgumentCaptor.forClass(BrowserLoginParams.class); + mockedAwsTelemetryProvider.verify(() -> AwsTelemetryProvider.emitLoginWithBrowserEvent(captor.capture())); + return captor.getValue(); + } + private LoginParams createLoginParams(final LoginIdcParams idcParams) { LoginParams loginParams = mock(LoginParams.class); when(loginParams.getLoginIdcParams()).thenReturn(idcParams); @@ -459,7 +632,12 @@ private AuthState createAuthState(final AuthStateType authStateType, final Login private void invokeProcessLogin(final LoginType loginType, final LoginParams loginParams, final boolean loginOnInvalidToken) throws Exception { - Object processLoginFuture = loginService.processLogin(loginType, loginParams, loginOnInvalidToken); + invokeProcessLogin(loginType, loginParams, loginOnInvalidToken, false); + } + + private void invokeProcessLogin(final LoginType loginType, final LoginParams loginParams, + final boolean loginOnInvalidToken, final boolean isReAuth) throws Exception { + Object processLoginFuture = loginService.processLogin(loginType, loginParams, loginOnInvalidToken, isReAuth); assertTrue(processLoginFuture instanceof CompletableFuture, "Return value should be CompletableFuture"); CompletableFuture future = (CompletableFuture) processLoginFuture; diff --git a/plugin/tst/software/aws/toolkits/eclipse/amazonq/providers/assets/ToolkitLoginWebViewAssetProviderTest.java b/plugin/tst/software/aws/toolkits/eclipse/amazonq/providers/assets/ToolkitLoginWebViewAssetProviderTest.java new file mode 100644 index 000000000..99e6d0b05 --- /dev/null +++ b/plugin/tst/software/aws/toolkits/eclipse/amazonq/providers/assets/ToolkitLoginWebViewAssetProviderTest.java @@ -0,0 +1,75 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.aws.toolkits.eclipse.amazonq.providers.assets; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mockConstruction; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.net.URL; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; + +import software.aws.toolkits.eclipse.amazonq.extensions.implementation.ActivatorStaticMockExtension; +import software.aws.toolkits.eclipse.amazonq.telemetry.ToolkitTelemetryProvider; +import software.aws.toolkits.eclipse.amazonq.util.PluginUtils; +import software.aws.toolkits.eclipse.amazonq.util.ThreadingUtils; +import software.aws.toolkits.eclipse.amazonq.util.WebviewAssetServer; +import software.aws.toolkits.telemetry.TelemetryDefinitions.Result; + +public final class ToolkitLoginWebViewAssetProviderTest { + + @RegisterExtension + private static ActivatorStaticMockExtension activatorStaticMockExtension = new ActivatorStaticMockExtension(); + + @Test + void initializeEmitsDidLoadLoginModuleFailedWhenAssetsAreMissing() { + try (MockedStatic mockedThreadingUtils = mockStatic(ThreadingUtils.class); + MockedStatic mockedPluginUtils = mockStatic(PluginUtils.class); + MockedStatic mockedToolkitTelemetryProvider = mockStatic(ToolkitTelemetryProvider.class)) { + mockedPluginUtils.when(() -> PluginUtils.getResource("webview/build/assets/js/getStart.js")) + .thenThrow(new IOException("resource is unavailable")); + + ToolkitLoginWebViewAssetProvider assetProvider = new ToolkitLoginWebViewAssetProvider(); + assetProvider.initialize(); + + ArgumentCaptor captor = ArgumentCaptor.forClass(Runnable.class); + mockedThreadingUtils.verify(() -> ThreadingUtils.executeAsyncTask(captor.capture())); + captor.getValue().run(); + + mockedToolkitTelemetryProvider.verify(() -> ToolkitTelemetryProvider.emitDidLoadModuleEventMetric( + ToolkitTelemetryProvider.LOGIN_MODULE, Result.FAILED, "DependencyMissing"), times(1)); + } + } + + @Test + void initializeEmitsDidLoadLoginModuleFailedWhenAssetsCannotBeServed() throws Exception { + try (MockedStatic mockedThreadingUtils = mockStatic(ThreadingUtils.class); + MockedStatic mockedPluginUtils = mockStatic(PluginUtils.class); + MockedConstruction mockedWebviewAssetServer = mockConstruction(WebviewAssetServer.class, + (mock, context) -> when(mock.resolve(any(String.class))).thenReturn(false)); + MockedStatic mockedToolkitTelemetryProvider = mockStatic(ToolkitTelemetryProvider.class)) { + mockedPluginUtils.when(() -> PluginUtils.getResource("webview/build/assets/js/getStart.js")) + .thenReturn(new URL("file:/assets/js/getStart.js")); + + ToolkitLoginWebViewAssetProvider assetProvider = new ToolkitLoginWebViewAssetProvider(); + assetProvider.initialize(); + + ArgumentCaptor captor = ArgumentCaptor.forClass(Runnable.class); + mockedThreadingUtils.verify(() -> ThreadingUtils.executeAsyncTask(captor.capture())); + captor.getValue().run(); + + mockedToolkitTelemetryProvider.verify(() -> ToolkitTelemetryProvider.emitDidLoadModuleEventMetric( + ToolkitTelemetryProvider.LOGIN_MODULE, Result.FAILED, "AssetLoadFailed"), times(1)); + } + } + +} diff --git a/plugin/tst/software/aws/toolkits/eclipse/amazonq/telemetry/AwsTelemetryProviderTest.java b/plugin/tst/software/aws/toolkits/eclipse/amazonq/telemetry/AwsTelemetryProviderTest.java new file mode 100644 index 000000000..03ab906af --- /dev/null +++ b/plugin/tst/software/aws/toolkits/eclipse/amazonq/telemetry/AwsTelemetryProviderTest.java @@ -0,0 +1,86 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.aws.toolkits.eclipse.amazonq.telemetry; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.verify; + +import java.util.Optional; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.mockito.ArgumentCaptor; + +import software.amazon.awssdk.services.toolkittelemetry.model.MetadataEntry; +import software.amazon.awssdk.services.toolkittelemetry.model.MetricDatum; +import software.aws.toolkits.eclipse.amazonq.extensions.implementation.ActivatorStaticMockExtension; +import software.aws.toolkits.eclipse.amazonq.telemetry.AwsTelemetryProvider.BrowserLoginParams; +import software.aws.toolkits.eclipse.amazonq.telemetry.service.TelemetryService; +import software.aws.toolkits.eclipse.amazonq.util.Constants; +import software.aws.toolkits.telemetry.TelemetryDefinitions.CredentialType; +import software.aws.toolkits.telemetry.TelemetryDefinitions.Result; + +public final class AwsTelemetryProviderTest { + + private static final String SESSION_DURATION_KEY = "sessionDuration"; + + @RegisterExtension + private static ActivatorStaticMockExtension activatorStaticMockExtension = new ActivatorStaticMockExtension(); + + @Test + void emitLoginWithBrowserEventReportsSessionDurationWhenKnown() { + AwsTelemetryProvider.emitLoginWithBrowserEvent(new BrowserLoginParams(Constants.AWS_BUILDER_ID_URL, + CredentialType.BEARER_TOKEN, false, Result.SUCCEEDED, null, 7776000000L)); + + MetricDatum metricDatum = captureMetricDatum(); + assertEquals("aws_loginWithBrowser", metricDatum.metricName()); + assertEquals(Optional.of("7776000000"), getMetadataValue(metricDatum, SESSION_DURATION_KEY)); + assertEquals(Optional.of(Constants.AWS_BUILDER_ID_URL), getMetadataValue(metricDatum, "credentialStartUrl")); + assertEquals(Optional.of("bearerToken"), getMetadataValue(metricDatum, "credentialType")); + assertEquals(Optional.of("Succeeded"), getMetadataValue(metricDatum, "result")); + assertEquals(Optional.of("false"), getMetadataValue(metricDatum, "isReAuth")); + assertFalse(metricDatum.passive()); + } + + @Test + void emitLoginWithBrowserEventLeavesOutSessionDurationWhenUnknown() { + AwsTelemetryProvider.emitLoginWithBrowserEvent(new BrowserLoginParams(Constants.AWS_BUILDER_ID_URL, + CredentialType.BEARER_TOKEN, false, Result.SUCCEEDED, null, null)); + + MetricDatum metricDatum = captureMetricDatum(); + assertTrue(getMetadataValue(metricDatum, SESSION_DURATION_KEY).isEmpty(), + "session duration should not be reported when no previous login is known"); + assertEquals(Optional.of(Constants.AWS_BUILDER_ID_URL), getMetadataValue(metricDatum, "credentialStartUrl")); + } + + @Test + void emitLoginWithBrowserEventReportsReasonCodeOnFailure() { + AwsTelemetryProvider.emitLoginWithBrowserEvent(new BrowserLoginParams(Constants.AWS_BUILDER_ID_URL, + CredentialType.BEARER_TOKEN, true, Result.FAILED, "IllegalStateException", null)); + + MetricDatum metricDatum = captureMetricDatum(); + assertEquals(Optional.of("Failed"), getMetadataValue(metricDatum, "result")); + assertEquals(Optional.of("IllegalStateException"), getMetadataValue(metricDatum, "reason")); + assertEquals(Optional.of("true"), getMetadataValue(metricDatum, "isReAuth")); + assertTrue(getMetadataValue(metricDatum, SESSION_DURATION_KEY).isEmpty(), + "session duration should not be reported for a failed login"); + } + + private MetricDatum captureMetricDatum() { + TelemetryService telemetryService = activatorStaticMockExtension.getMock(TelemetryService.class); + ArgumentCaptor captor = ArgumentCaptor.forClass(MetricDatum.class); + verify(telemetryService).emitMetric(captor.capture()); + return captor.getValue(); + } + + private Optional getMetadataValue(final MetricDatum metricDatum, final String key) { + return metricDatum.metadata().stream() + .filter(entry -> key.equals(entry.key())) + .map(MetadataEntry::value) + .findFirst(); + } + +} diff --git a/plugin/tst/software/aws/toolkits/eclipse/amazonq/telemetry/ToolkitTelemetryProviderTest.java b/plugin/tst/software/aws/toolkits/eclipse/amazonq/telemetry/ToolkitTelemetryProviderTest.java new file mode 100644 index 000000000..6656acbf5 --- /dev/null +++ b/plugin/tst/software/aws/toolkits/eclipse/amazonq/telemetry/ToolkitTelemetryProviderTest.java @@ -0,0 +1,63 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.aws.toolkits.eclipse.amazonq.telemetry; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.verify; + +import java.util.Optional; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.mockito.ArgumentCaptor; + +import software.amazon.awssdk.services.toolkittelemetry.model.MetadataEntry; +import software.amazon.awssdk.services.toolkittelemetry.model.MetricDatum; +import software.aws.toolkits.eclipse.amazonq.extensions.implementation.ActivatorStaticMockExtension; +import software.aws.toolkits.eclipse.amazonq.telemetry.service.TelemetryService; +import software.aws.toolkits.telemetry.TelemetryDefinitions.Result; + +public final class ToolkitTelemetryProviderTest { + + @RegisterExtension + private static ActivatorStaticMockExtension activatorStaticMockExtension = new ActivatorStaticMockExtension(); + + @Test + void emitDidLoadModuleEventMetricReportsLoginModuleLoaded() { + ToolkitTelemetryProvider.emitDidLoadModuleEventMetric(ToolkitTelemetryProvider.LOGIN_MODULE, Result.SUCCEEDED, null); + + MetricDatum metricDatum = captureMetricDatum(); + assertEquals("toolkit_didLoadModule", metricDatum.metricName()); + assertEquals(Optional.of("login"), getMetadataValue(metricDatum, "module")); + assertEquals(Optional.of("Succeeded"), getMetadataValue(metricDatum, "result")); + assertTrue(metricDatum.passive()); + } + + @Test + void emitDidLoadModuleEventMetricReportsReasonCodeOnFailure() { + ToolkitTelemetryProvider.emitDidLoadModuleEventMetric(ToolkitTelemetryProvider.LOGIN_MODULE, Result.FAILED, + "DependencyMissing"); + + MetricDatum metricDatum = captureMetricDatum(); + assertEquals(Optional.of("login"), getMetadataValue(metricDatum, "module")); + assertEquals(Optional.of("Failed"), getMetadataValue(metricDatum, "result")); + assertEquals(Optional.of("DependencyMissing"), getMetadataValue(metricDatum, "reason")); + } + + private MetricDatum captureMetricDatum() { + TelemetryService telemetryService = activatorStaticMockExtension.getMock(TelemetryService.class); + ArgumentCaptor captor = ArgumentCaptor.forClass(MetricDatum.class); + verify(telemetryService).emitMetric(captor.capture()); + return captor.getValue(); + } + + private Optional getMetadataValue(final MetricDatum metricDatum, final String key) { + return metricDatum.metadata().stream() + .filter(entry -> key.equals(entry.key())) + .map(MetadataEntry::value) + .findFirst(); + } + +}