diff --git a/plugin/codegen-resources/definitions/commonDefinitions.json b/plugin/codegen-resources/definitions/commonDefinitions.json index 96044d8d1..8cdfe7e90 100644 --- a/plugin/codegen-resources/definitions/commonDefinitions.json +++ b/plugin/codegen-resources/definitions/commonDefinitions.json @@ -2889,9 +2889,17 @@ "type": "reAuthReason", "required": false }, + { + "type": "reason", + "required": false + }, { "type": "result" }, + { + "type": "sessionDuration", + "required": false + }, { "type": "source" }, @@ -7061,6 +7069,37 @@ ], "passive": true }, + { + "name": "toolkit_didLoadModule", + "description": "The module has loaded, i.e it has rendered/resolved/finished. You can use this metric by itself, OR after `toolkit_willOpenModule` + `traceId` to close the loop on an asynchronous operation.", + "metadata": [ + { + "type": "attempts", + "required": false + }, + { + "type": "duration", + "required": false + }, + { + "type": "module", + "required": true + }, + { + "type": "reason", + "required": false + }, + { + "type": "result", + "required": true + }, + { + "type": "version", + "required": false + } + ], + "passive": true + }, { "name": "toolkit_execute", "description": "Emitted whenever a registered command is executed", diff --git a/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/AuthPluginStore.java b/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/AuthPluginStore.java index ff25fc109..7f08a0185 100644 --- a/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/AuthPluginStore.java +++ b/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/AuthPluginStore.java @@ -3,6 +3,9 @@ package software.aws.toolkits.eclipse.amazonq.lsp.auth; +import java.time.Instant; +import java.util.Optional; + import software.aws.toolkits.eclipse.amazonq.configuration.PluginStore; import software.aws.toolkits.eclipse.amazonq.lsp.auth.model.LoginIdcParams; import software.aws.toolkits.eclipse.amazonq.lsp.auth.model.LoginParams; @@ -54,9 +57,49 @@ public String getSsoTokenId() { return pluginStore.get(Constants.SSO_TOKEN_ID); } + /** + * Persists the instant of the latest successful login alongside the start url it was performed + * against. It is used to report how long the previous authentication session lived for. + * + * @param startUrl the start url the login was performed against + * @param loginInstant the instant the login completed + */ + public void setLoginTimestamp(final String startUrl, final Instant loginInstant) { + if (startUrl == null || loginInstant == null) { + return; + } + pluginStore.put(Constants.LOGIN_TIMESTAMP_START_URL_KEY, startUrl); + pluginStore.put(Constants.LOGIN_TIMESTAMP_KEY, String.valueOf(loginInstant.toEpochMilli())); + } + + /** + * Retrieves the instant of the latest successful login for the given start url. An empty value is + * returned when no login has been recorded yet, when the recorded login was performed against a + * different start url, or when the recorded value cannot be parsed. + * + * @param startUrl the start url the login is being performed against + * @return the instant of the previous successful login for that start url + */ + public Optional getLoginTimestamp(final String startUrl) { + String storedStartUrl = pluginStore.get(Constants.LOGIN_TIMESTAMP_START_URL_KEY); + String storedTimestamp = pluginStore.get(Constants.LOGIN_TIMESTAMP_KEY); + + if (startUrl == null || storedStartUrl == null || storedTimestamp == null || !storedStartUrl.equals(startUrl)) { + return Optional.empty(); + } + + try { + return Optional.of(Instant.ofEpochMilli(Long.parseLong(storedTimestamp))); + } catch (NumberFormatException ex) { + return Optional.empty(); + } + } + public void clear() { pluginStore.remove(Constants.LOGIN_TYPE_KEY); pluginStore.remove(Constants.LOGIN_IDC_PARAMS_KEY); + pluginStore.remove(Constants.LOGIN_TIMESTAMP_START_URL_KEY); + pluginStore.remove(Constants.LOGIN_TIMESTAMP_KEY); pluginStore.remove(Constants.SSO_TOKEN_ID); } } diff --git a/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultAuthStateManager.java b/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultAuthStateManager.java index 421b6f1c7..75a13b2db 100644 --- a/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultAuthStateManager.java +++ b/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultAuthStateManager.java @@ -10,7 +10,9 @@ import software.aws.toolkits.eclipse.amazonq.lsp.auth.model.LoginParams; import software.aws.toolkits.eclipse.amazonq.lsp.auth.model.LoginType; import software.aws.toolkits.eclipse.amazonq.plugin.Activator; +import software.aws.toolkits.eclipse.amazonq.telemetry.AuthTelemetryProvider; import software.aws.toolkits.eclipse.amazonq.util.AuthUtil; +import software.aws.toolkits.telemetry.TelemetryDefinitions.AuthStatus; /** * Manages authentication state transitions and persistence in the Amazon Q plugin. @@ -43,6 +45,8 @@ public final class DefaultAuthStateManager implements AuthStateManager { private String issuerUrl; // used in AmazonQLspClientImpl.getConnectionMetadata() private String ssoTokenId; // used in logout's invalidateSsoToken params private AuthState previousAuthState = null; + private boolean isRestoringPersistedAuthState = false; + private boolean hasEmittedStartupAuthState = false; public DefaultAuthStateManager(final PluginStore pluginStore) { this.authPluginStore = new AuthPluginStore(pluginStore); @@ -132,6 +136,43 @@ private void updateState(final AuthStateType authStatusType, final LoginType log } } previousAuthState = newAuthState; + + emitStartupAuthStateMetric(newAuthState); + } + + /** + * Reports the authentication state observed at startup, once per plugin session. + * + * The state restored from the plugin store is optimistic: a stored connection is assumed to still be + * valid until the re-authentication performed on start up resolves it. The optimistic state is + * therefore skipped and the metric is reported for the state that follows it, which is the outcome + * of that re-authentication. A restored logged out state needs no re-authentication and is + * definitive right away. + * + * @param authState the state the plugin transitioned to + * @see #syncAuthStateWithPluginStore() + * @see DefaultLoginService + */ + private void emitStartupAuthStateMetric(final AuthState authState) { + if (hasEmittedStartupAuthState || isRestoringPersistedAuthState) { + return; + } + hasEmittedStartupAuthState = true; + + AuthTelemetryProvider.emitUserStateOnStartupMetric(toAuthStatus(authState.authStateType()), authState.issuerUrl()); + } + + private static AuthStatus toAuthStatus(final AuthStateType authStateType) { + switch (authStateType) { + case LOGGED_IN: + return AuthStatus.CONNECTED; + case EXPIRED: + return AuthStatus.EXPIRED; + case LOGGED_OUT: + return AuthStatus.NOT_CONNECTED; + default: + return AuthStatus.UNKNOWN; + } } private void syncAuthStateWithPluginStore() { @@ -162,10 +203,18 @@ private void syncAuthStateWithPluginStore() { * * @see DefaultLoginService constructor that handles the re-authentication on LoginService start up */ + boolean restoreFailed = false; try { + isRestoringPersistedAuthState = true; toLoggedIn(loginType, loginParams, ssoTokenId); } catch (Exception ex) { Activator.getLogger().error("Failed to transition to a logged in state after syncing auth state with the persistent store", ex); + restoreFailed = true; + } finally { + isRestoringPersistedAuthState = false; + } + + if (restoreFailed) { toLoggedOut(); } } diff --git a/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultLoginService.java b/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultLoginService.java index f2f9df388..225420c90 100644 --- a/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultLoginService.java +++ b/plugin/src/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultLoginService.java @@ -3,8 +3,11 @@ package software.aws.toolkits.eclipse.amazonq.lsp.auth; +import java.time.Duration; +import java.time.Instant; import java.util.Objects; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.atomic.AtomicReference; import software.aws.toolkits.eclipse.amazonq.configuration.PluginStore; @@ -18,7 +21,11 @@ import software.aws.toolkits.eclipse.amazonq.lsp.model.UpdateCredentialsPayload; import software.aws.toolkits.eclipse.amazonq.plugin.Activator; import software.aws.toolkits.eclipse.amazonq.providers.lsp.LspProvider; +import software.aws.toolkits.eclipse.amazonq.telemetry.AwsTelemetryProvider; +import software.aws.toolkits.eclipse.amazonq.telemetry.AwsTelemetryProvider.BrowserLoginParams; import software.aws.toolkits.eclipse.amazonq.util.AuthUtil; +import software.aws.toolkits.telemetry.TelemetryDefinitions.CredentialType; +import software.aws.toolkits.telemetry.TelemetryDefinitions.Result; /** * Core authentication service for the Amazon Q Eclipse plugin that manages @@ -41,11 +48,13 @@ public final class DefaultLoginService implements LoginService { private AuthStateManager authStateManager; private AuthTokenService authTokenService; private AuthCredentialsService authCredentialsService; + private AuthPluginStore authPluginStore; private DefaultLoginService(final Builder builder) { this.authStateManager = Objects.requireNonNull(builder.authStateManager, "authStateManager cannot be null"); this.authTokenService = Objects.requireNonNull(builder.authTokenService, "authTokenService cannot be null"); this.authCredentialsService = Objects.requireNonNull(builder.authCredentialsService, "authCredentialsService cannot be null"); + this.authPluginStore = new AuthPluginStore(Objects.requireNonNull(builder.pluginStore, "pluginStore cannot be null")); if (builder.initializeOnStartUp) { AuthState authState = authStateManager.getAuthState(); @@ -73,7 +82,7 @@ public CompletableFuture login(final LoginType loginType, final LoginParam Activator.getLogger().info("Attempting to login..."); - return processLogin(loginType, loginParams, true) + return processLogin(loginType, loginParams, true, false) .exceptionally(throwable -> { Activator.getLogger().error("Failed to log in", throwable); logout(); @@ -141,7 +150,7 @@ public CompletableFuture reAuthenticate(final boolean loginOnInvalidToken) Activator.getLogger().info("Attempting to re-authenticate..."); - return processLogin(authState.loginType(), authState.loginParams(), loginOnInvalidToken) + return processLogin(authState.loginType(), authState.loginParams(), loginOnInvalidToken, true) .exceptionally(throwable -> { Activator.getLogger().error("Failed to re-authenticate", throwable); logout(); @@ -154,7 +163,8 @@ public AuthState getAuthState() { return authStateManager.getAuthState(); } - CompletableFuture processLogin(final LoginType loginType, final LoginParams loginParams, final boolean loginOnInvalidToken) { + CompletableFuture processLogin(final LoginType loginType, final LoginParams loginParams, final boolean loginOnInvalidToken, + final boolean isReAuth) { AuthUtil.validateLoginParameters(loginType, loginParams); final AtomicReference ssoTokenId = new AtomicReference<>(); // Saved for logout @@ -172,7 +182,24 @@ CompletableFuture processLogin(final LoginType loginType, final LoginParam }) .thenRun(() -> { authStateManager.toLoggedIn(loginType, loginParams, ssoTokenId.get()); + if (loginOnInvalidToken) { + emitBrowserLoginMetric(loginType, loginParams, isReAuth, Result.SUCCEEDED, null); + } Activator.getLogger().info("Successfully logged in"); + }) + /* + * Reports the outcome of the login itself. The steps that follow are not part of the login, + * so they are wired after this stage to keep them out of the metric. + * + * Only logins that were allowed to open the browser are reported. The re-authentication + * performed on start up passes loginOnInvalidToken=false, it refreshes the cached token + * silently and would otherwise report a browser login (and a session duration) on every + * start of the IDE. + */ + .whenComplete((unused, throwable) -> { + if (throwable != null && loginOnInvalidToken) { + emitBrowserLoginMetric(loginType, loginParams, isReAuth, Result.FAILED, getReasonCode(throwable)); + } }).thenRun(() -> { CustomizationUtil.triggerChangeConfigurationNotification(); }).exceptionally(throwable -> { @@ -180,6 +207,50 @@ CompletableFuture processLogin(final LoginType loginType, final LoginParam }); } + /** + * Emits the browser login metric, reporting how long the previous authentication session for the + * same start url lived for. + * + * The session duration is only known once a login has been recorded for that start url, so it is + * left out of the first login and of the first login that follows a sign out, which clears the + * recorded login. A successful login becomes the new reference point for the next one. + * + * @param loginType the type of connection being authenticated + * @param loginParams the parameters of the connection being authenticated + * @param isReAuth whether the login renews an existing connection + * @param result whether the login succeeded + * @param reason a short reason code when the login failed, null otherwise + */ + private void emitBrowserLoginMetric(final LoginType loginType, final LoginParams loginParams, final boolean isReAuth, + final Result result, final String reason) { + String credentialStartUrl = AuthUtil.getIssuerUrl(loginType, loginParams); + + // The start url identifies the authentication session, a metric without it carries no signal. + if (credentialStartUrl == null || credentialStartUrl.isBlank()) { + return; + } + + Long sessionDuration = null; + if (result == Result.SUCCEEDED) { + Instant loginInstant = Instant.now(); + sessionDuration = authPluginStore.getLoginTimestamp(credentialStartUrl) + .map(previousLogin -> Duration.between(previousLogin, loginInstant).toMillis()) + .filter(duration -> duration >= 0) // guards against a recorded login dated in the future + .orElse(null); + authPluginStore.setLoginTimestamp(credentialStartUrl, loginInstant); + } + + AwsTelemetryProvider.emitLoginWithBrowserEvent(new BrowserLoginParams(credentialStartUrl, + CredentialType.BEARER_TOKEN, isReAuth, result, reason, sessionDuration)); + } + + private static String getReasonCode(final Throwable throwable) { + Throwable cause = throwable instanceof CompletionException && throwable.getCause() != null + ? throwable.getCause() + : throwable; + return cause.getClass().getSimpleName(); + } + public static class Builder { private LspProvider lspProvider; private PluginStore pluginStore; diff --git a/plugin/src/software/aws/toolkits/eclipse/amazonq/providers/assets/ToolkitLoginWebViewAssetProvider.java b/plugin/src/software/aws/toolkits/eclipse/amazonq/providers/assets/ToolkitLoginWebViewAssetProvider.java index 343d6ec79..372306395 100644 --- a/plugin/src/software/aws/toolkits/eclipse/amazonq/providers/assets/ToolkitLoginWebViewAssetProvider.java +++ b/plugin/src/software/aws/toolkits/eclipse/amazonq/providers/assets/ToolkitLoginWebViewAssetProvider.java @@ -16,6 +16,7 @@ import software.aws.toolkits.eclipse.amazonq.broker.events.ToolkitLoginWebViewAssetState; import software.aws.toolkits.eclipse.amazonq.plugin.Activator; +import software.aws.toolkits.eclipse.amazonq.telemetry.ToolkitTelemetryProvider; import software.aws.toolkits.eclipse.amazonq.telemetry.UiTelemetryProvider; import software.aws.toolkits.eclipse.amazonq.util.PluginUtils; import software.aws.toolkits.eclipse.amazonq.util.ThemeDetector; @@ -26,9 +27,13 @@ import software.aws.toolkits.eclipse.amazonq.views.ViewActionHandler; import software.aws.toolkits.eclipse.amazonq.views.ViewCommandParser; import software.aws.toolkits.eclipse.amazonq.views.ViewConstants; +import software.aws.toolkits.telemetry.TelemetryDefinitions.Result; public final class ToolkitLoginWebViewAssetProvider extends WebViewAssetProvider { + private static final String DEPENDENCY_MISSING_REASON = "DependencyMissing"; + private static final String ASSET_LOAD_FAILED_REASON = "AssetLoadFailed"; + private WebviewAssetServer webviewAssetServer; private static final ThemeDetector THEME_DETECTOR = new ThemeDetector(); private final ViewCommandParser commandParser; @@ -47,6 +52,10 @@ public void initialize() { if (content.isEmpty()) { ThreadingUtils.executeAsyncTask(() -> { content = resolveContent(); + if (content.isEmpty()) { + ToolkitTelemetryProvider.emitDidLoadModuleEventMetric(ToolkitTelemetryProvider.LOGIN_MODULE, + Result.FAILED, DEPENDENCY_MISSING_REASON); + } Activator.getEventBroker().post(ToolkitLoginWebViewAssetState.class, content.isPresent() ? ToolkitLoginWebViewAssetState.RESOLVED : ToolkitLoginWebViewAssetState.DEPENDENCY_MISSING); @@ -93,6 +102,8 @@ private Optional resolveContent() { webviewAssetServer = new WebviewAssetServer(); var result = webviewAssetServer.resolve(jsDirectoryPath); if (!result) { + ToolkitTelemetryProvider.emitDidLoadModuleEventMetric(ToolkitTelemetryProvider.LOGIN_MODULE, + Result.FAILED, ASSET_LOAD_FAILED_REASON); return Optional.of("Failed to load JS"); } var loginJsPath = webviewAssetServer.getUri() + "getStart.js"; @@ -146,6 +157,8 @@ private Optional resolveContent() { """, loginJsPath, loginJsPath, loginJsPath, getWaitFunction(), isDarkTheme)); } catch (IOException e) { + ToolkitTelemetryProvider.emitDidLoadModuleEventMetric(ToolkitTelemetryProvider.LOGIN_MODULE, + Result.FAILED, DEPENDENCY_MISSING_REASON); return Optional.of("Failed to load JS"); } } diff --git a/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/AuthTelemetryProvider.java b/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/AuthTelemetryProvider.java new file mode 100644 index 000000000..82b8c0f9a --- /dev/null +++ b/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/AuthTelemetryProvider.java @@ -0,0 +1,39 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.aws.toolkits.eclipse.amazonq.telemetry; + +import java.time.Instant; + +import software.amazon.awssdk.services.toolkittelemetry.model.MetricDatum; +import software.aws.toolkits.eclipse.amazonq.plugin.Activator; +import software.aws.toolkits.telemetry.AuthTelemetry; +import software.aws.toolkits.telemetry.TelemetryDefinitions.AuthStatus; + +public final class AuthTelemetryProvider { + + private static final String STARTUP_SOURCE = "startup"; + + private AuthTelemetryProvider() { + //prevent instantiation + } + + /** + * Reports the authentication state observed when the plugin starts. + * + * @param authStatus the authentication state the plugin started in + * @param credentialStartUrl the start url of the restored connection, null when there is none + */ + public static void emitUserStateOnStartupMetric(final AuthStatus authStatus, final String credentialStartUrl) { + MetricDatum metricDatum = AuthTelemetry.UserStateEvent() + .authStatus(authStatus) + .credentialStartUrl(credentialStartUrl) + .source(STARTUP_SOURCE) + .passive(true) + .createTime(Instant.now()) + .value(1.0) + .build(); + Activator.getTelemetryService().emitMetric(metricDatum); + } + +} diff --git a/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/AwsTelemetryProvider.java b/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/AwsTelemetryProvider.java index 89d382fed..99679ed1f 100644 --- a/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/AwsTelemetryProvider.java +++ b/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/AwsTelemetryProvider.java @@ -4,13 +4,22 @@ package software.aws.toolkits.eclipse.amazonq.telemetry; import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import software.amazon.awssdk.services.toolkittelemetry.model.MetadataEntry; import software.amazon.awssdk.services.toolkittelemetry.model.MetricDatum; import software.aws.toolkits.eclipse.amazonq.plugin.Activator; import software.aws.toolkits.telemetry.AwsTelemetry; +import software.aws.toolkits.telemetry.TelemetryDefinitions.CredentialType; +import software.aws.toolkits.telemetry.TelemetryDefinitions.Result; public final class AwsTelemetryProvider { + private static final String SESSION_DURATION_KEY = "sessionDuration"; + private static final String AUTH_VIEW_SOURCE = "authView"; + private static final String RE_AUTH_SOURCE = "reAuth"; + private AwsTelemetryProvider() { //prevent instantiation } @@ -26,4 +35,61 @@ public static void emitModifySettingEvent(final String settingId, final String s Activator.getTelemetryService().emitMetric(metricDatum); } + public static void emitLoginWithBrowserEvent(final BrowserLoginParams params) { + MetricDatum metricDatum = AwsTelemetry.LoginWithBrowserEvent() + .credentialStartUrl(params.credentialStartUrl()) + .credentialType(params.credentialType()) + .isReAuth(params.isReAuth()) + .result(params.result()) + .reason(params.reason()) + .source(params.isReAuth() ? RE_AUTH_SOURCE : AUTH_VIEW_SOURCE) + .passive(false) + .createTime(Instant.now()) + .value(1.0) + .build(); + Activator.getTelemetryService().emitMetric(withSessionDuration(metricDatum, params.sessionDuration())); + } + + /** + * Rewrites the session duration metadata entry of the given metric. + * + * The generated builder types session duration as a primitive int, which serializes to zero when + * it is not set and cannot hold durations longer than roughly twenty five days in milliseconds. + * Authentication sessions are expected to live for months, and a first login has no previous + * session at all, so the entry is written with the exact millisecond value when a previous session + * is known and removed when it is not. + * + * @param metricDatum the metric built by the generated builder + * @param sessionDuration milliseconds since the previous successful login, or null when unknown + * @return the metric carrying an accurate session duration, or none at all + */ + private static MetricDatum withSessionDuration(final MetricDatum metricDatum, final Long sessionDuration) { + List metadata = new ArrayList<>(); + for (MetadataEntry entry : metricDatum.metadata()) { + if (!SESSION_DURATION_KEY.equals(entry.key())) { + metadata.add(entry); + } + } + if (sessionDuration != null) { + metadata.add(MetadataEntry.builder() + .key(SESSION_DURATION_KEY) + .value(String.valueOf(sessionDuration)) + .build()); + } + return metricDatum.toBuilder().metadata(metadata).build(); + } + + /** + * Parameters of the browser login metric. + * + * @param credentialStartUrl the start url the login was performed against + * @param credentialType the type of credentials the login produces + * @param isReAuth whether the login renewed an existing connection + * @param result whether the login succeeded + * @param reason a short reason code when the login failed, null otherwise + * @param sessionDuration milliseconds since the previous successful login, null when unknown + */ + public record BrowserLoginParams(String credentialStartUrl, CredentialType credentialType, boolean isReAuth, + Result result, String reason, Long sessionDuration) { }; + } diff --git a/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/ToolkitTelemetryProvider.java b/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/ToolkitTelemetryProvider.java index 6724a6311..fa4aa1042 100644 --- a/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/ToolkitTelemetryProvider.java +++ b/plugin/src/software/aws/toolkits/eclipse/amazonq/telemetry/ToolkitTelemetryProvider.java @@ -11,6 +11,8 @@ import java.util.Set; public final class ToolkitTelemetryProvider { + public static final String LOGIN_MODULE = "login"; + private static final Set NON_PASSIVE = Set.of("ellipsesMenu", "statusBar", "shortcut"); private ToolkitTelemetryProvider() { @@ -59,6 +61,27 @@ public static void emitCloseModuleEventMetric(final String module, final String .build(); Activator.getTelemetryService().emitMetric(metadata); } + + /** + * Reports that a module finished loading, or failed to load. The module names are shared with the + * other Amazon Q IDE plugins, so pass one of the constants declared on this class. + * + * @param module the module that finished loading + * @param result whether the module loaded + * @param reason a short reason code when the module failed to load, null otherwise + */ + public static void emitDidLoadModuleEventMetric(final String module, final Result result, final String reason) { + MetricDatum metadata = ToolkitTelemetry.DidLoadModuleEvent() + .module(module) + .result(result) + .reason(reason) + .passive(true) + .createTime(Instant.now()) + .value(1.0) + .build(); + Activator.getTelemetryService().emitMetric(metadata); + } + private static String mapModuleId(final String viewId) { String page = viewId.substring(viewId.lastIndexOf(".") + 1); switch (page) { diff --git a/plugin/src/software/aws/toolkits/eclipse/amazonq/util/Constants.java b/plugin/src/software/aws/toolkits/eclipse/amazonq/util/Constants.java index 82294da69..da26120c8 100644 --- a/plugin/src/software/aws/toolkits/eclipse/amazonq/util/Constants.java +++ b/plugin/src/software/aws/toolkits/eclipse/amazonq/util/Constants.java @@ -32,6 +32,8 @@ private Constants() { public static final String DEFAULT_Q_FOUNDATION_DISPLAY_NAME = "Amazon Q foundation (Default)"; public static final String LOGIN_TYPE_KEY = "LOGIN_TYPE"; public static final String LOGIN_IDC_PARAMS_KEY = "IDC_PARAMS"; + public static final String LOGIN_TIMESTAMP_KEY = "LOGIN_TIMESTAMP"; + public static final String LOGIN_TIMESTAMP_START_URL_KEY = "LOGIN_TIMESTAMP_START_URL"; public static final String SSO_TOKEN_ID = "SSO_TOKEN_IN"; public static final String AWS_BUILDER_ID_URL = "https://view.awsapps.com/start"; public static final String IDC_PROFILE_NAME = "eclipse-q-profile"; diff --git a/plugin/src/software/aws/toolkits/eclipse/amazonq/views/LoginViewActionHandler.java b/plugin/src/software/aws/toolkits/eclipse/amazonq/views/LoginViewActionHandler.java index 0581914ef..c0d1f91b1 100644 --- a/plugin/src/software/aws/toolkits/eclipse/amazonq/views/LoginViewActionHandler.java +++ b/plugin/src/software/aws/toolkits/eclipse/amazonq/views/LoginViewActionHandler.java @@ -20,6 +20,7 @@ import software.aws.toolkits.eclipse.amazonq.lsp.auth.model.LoginParams; import software.aws.toolkits.eclipse.amazonq.lsp.auth.model.LoginType; import software.aws.toolkits.eclipse.amazonq.plugin.Activator; +import software.aws.toolkits.eclipse.amazonq.telemetry.ToolkitTelemetryProvider; import software.aws.toolkits.eclipse.amazonq.util.AwsRegion; import software.aws.toolkits.eclipse.amazonq.util.JsonHandler; import software.aws.toolkits.eclipse.amazonq.util.PluginUtils; @@ -28,6 +29,7 @@ import software.aws.toolkits.eclipse.amazonq.views.model.Command; import software.aws.toolkits.eclipse.amazonq.views.model.ParsedCommand; import software.aws.toolkits.eclipse.amazonq.views.model.QDeveloperProfile; +import software.aws.toolkits.telemetry.TelemetryDefinitions.Result; public class LoginViewActionHandler implements ViewActionHandler { @@ -118,6 +120,8 @@ public final void handleCommand(final ParsedCommand parsedCommand, final Browser browser.execute("changeTheme(" + THEME_DETECTOR.isDarkTheme() + ");"); browser.execute(String.format("ideClient.prepareUi(%s)", js)); browser.execute("ideClient.updateAuthorization('')"); + // The login webview reports onLoad once its assets have finished loading. + ToolkitTelemetryProvider.emitDidLoadModuleEventMetric(ToolkitTelemetryProvider.LOGIN_MODULE, Result.SUCCEEDED, null); break; case ON_SELECT_PROFILE: QDeveloperProfile developerProfile = JSON_HANDLER.convertObject(params, QDeveloperProfile.class); diff --git a/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/AuthPluginStoreTest.java b/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/AuthPluginStoreTest.java index 648f1efb8..f5a84fa6f 100644 --- a/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/AuthPluginStoreTest.java +++ b/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/AuthPluginStoreTest.java @@ -5,9 +5,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; +import java.time.Instant; +import java.util.Optional; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; @@ -116,12 +121,71 @@ public void testGetSsoTokenId() { assertEquals(expectedToken, result); } + @Test + public void testSetLoginTimestamp() { + Instant loginInstant = Instant.ofEpochMilli(1700000000000L); + + authPluginStore.setLoginTimestamp("https://example.com", loginInstant); + + verify(pluginStore).put(Constants.LOGIN_TIMESTAMP_START_URL_KEY, "https://example.com"); + verify(pluginStore).put(Constants.LOGIN_TIMESTAMP_KEY, "1700000000000"); + } + + @Test + public void testSetLoginTimestampWithNullStartUrlDoesNotStore() { + authPluginStore.setLoginTimestamp(null, Instant.ofEpochMilli(1700000000000L)); + + verifyNoInteractions(pluginStore); + } + + @Test + public void testGetLoginTimestampForSameStartUrl() { + when(pluginStore.get(Constants.LOGIN_TIMESTAMP_START_URL_KEY)).thenReturn("https://example.com"); + when(pluginStore.get(Constants.LOGIN_TIMESTAMP_KEY)).thenReturn("1700000000000"); + + Optional result = authPluginStore.getLoginTimestamp("https://example.com"); + + assertEquals(Optional.of(Instant.ofEpochMilli(1700000000000L)), result); + } + + @Test + public void testGetLoginTimestampForDifferentStartUrlReturnsEmpty() { + when(pluginStore.get(Constants.LOGIN_TIMESTAMP_START_URL_KEY)).thenReturn("https://example.com"); + when(pluginStore.get(Constants.LOGIN_TIMESTAMP_KEY)).thenReturn("1700000000000"); + + Optional result = authPluginStore.getLoginTimestamp("https://other.example.com"); + + assertTrue(result.isEmpty()); + } + + @Test + public void testGetLoginTimestampWhenNoTimestampStoredReturnsEmpty() { + when(pluginStore.get(Constants.LOGIN_TIMESTAMP_START_URL_KEY)).thenReturn(null); + when(pluginStore.get(Constants.LOGIN_TIMESTAMP_KEY)).thenReturn(null); + + Optional result = authPluginStore.getLoginTimestamp("https://example.com"); + + assertTrue(result.isEmpty()); + } + + @Test + public void testGetLoginTimestampWhenTimestampNotANumberReturnsEmpty() { + when(pluginStore.get(Constants.LOGIN_TIMESTAMP_START_URL_KEY)).thenReturn("https://example.com"); + when(pluginStore.get(Constants.LOGIN_TIMESTAMP_KEY)).thenReturn("not-a-timestamp"); + + Optional result = authPluginStore.getLoginTimestamp("https://example.com"); + + assertTrue(result.isEmpty()); + } + @Test public void testClear() { authPluginStore.clear(); verify(pluginStore).remove(Constants.LOGIN_TYPE_KEY); verify(pluginStore).remove(Constants.LOGIN_IDC_PARAMS_KEY); + verify(pluginStore).remove(Constants.LOGIN_TIMESTAMP_START_URL_KEY); + verify(pluginStore).remove(Constants.LOGIN_TIMESTAMP_KEY); verify(pluginStore).remove(Constants.SSO_TOKEN_ID); } } diff --git a/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultAuthStateManagerTest.java b/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultAuthStateManagerTest.java index ee94f1315..43f357252 100644 --- a/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultAuthStateManagerTest.java +++ b/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultAuthStateManagerTest.java @@ -7,7 +7,10 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -18,6 +21,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.MockitoAnnotations; import software.aws.toolkits.eclipse.amazonq.configuration.PluginStore; @@ -28,7 +32,9 @@ import software.aws.toolkits.eclipse.amazonq.lsp.auth.model.LoginIdcParams; import software.aws.toolkits.eclipse.amazonq.lsp.auth.model.LoginParams; import software.aws.toolkits.eclipse.amazonq.lsp.auth.model.LoginType; +import software.aws.toolkits.eclipse.amazonq.telemetry.AuthTelemetryProvider; import software.aws.toolkits.eclipse.amazonq.util.Constants; +import software.aws.toolkits.telemetry.TelemetryDefinitions.AuthStatus; class DefaultAuthStateManagerTest { @@ -221,6 +227,69 @@ void syncAuthStateWithPluginStoreWithNoStoredCredentialsSetsLoggedOut() { assertNull(state.issuerUrl()); } + @Test + void syncAuthStateWithPluginStoreWithNoStoredCredentialsEmitsNotConnectedStartupUserState() { + when(pluginStore.get(Constants.LOGIN_TYPE_KEY)).thenReturn(LoginType.NONE.name()); + + try (MockedStatic mockedAuthTelemetryProvider = mockStatic(AuthTelemetryProvider.class)) { + new DefaultAuthStateManager(pluginStore); + + mockedAuthTelemetryProvider + .verify(() -> AuthTelemetryProvider.emitUserStateOnStartupMetric(AuthStatus.NOT_CONNECTED, null)); + } + } + + @Test + void syncAuthStateWithPluginStoreWithStoredCredentialsDefersStartupUserStateUntilReAuthenticationResolves() { + when(pluginStore.get(Constants.LOGIN_TYPE_KEY)).thenReturn(LoginType.BUILDER_ID.name()); + when(pluginStore.getObject(Constants.LOGIN_IDC_PARAMS_KEY, LoginIdcParams.class)) + .thenReturn(loginParams.getLoginIdcParams()); + when(pluginStore.get(Constants.SSO_TOKEN_ID)).thenReturn("ssoTokenId"); + + try (MockedStatic mockedAuthTelemetryProvider = mockStatic(AuthTelemetryProvider.class)) { + DefaultAuthStateManager newManager = new DefaultAuthStateManager(pluginStore); + + mockedAuthTelemetryProvider.verifyNoInteractions(); + + newManager.toLoggedIn(LoginType.BUILDER_ID, loginParams, "ssoTokenId"); + + mockedAuthTelemetryProvider.verify(() -> AuthTelemetryProvider + .emitUserStateOnStartupMetric(AuthStatus.CONNECTED, Constants.AWS_BUILDER_ID_URL)); + } + } + + @Test + void startupUserStateReportsExpiredWhenRestoredSessionExpires() { + when(pluginStore.get(Constants.LOGIN_TYPE_KEY)).thenReturn(LoginType.BUILDER_ID.name()); + when(pluginStore.getObject(Constants.LOGIN_IDC_PARAMS_KEY, LoginIdcParams.class)) + .thenReturn(loginParams.getLoginIdcParams()); + when(pluginStore.get(Constants.SSO_TOKEN_ID)).thenReturn("ssoTokenId"); + + try (MockedStatic mockedAuthTelemetryProvider = mockStatic(AuthTelemetryProvider.class)) { + DefaultAuthStateManager newManager = new DefaultAuthStateManager(pluginStore); + + newManager.toExpired(); + + mockedAuthTelemetryProvider.verify(() -> AuthTelemetryProvider + .emitUserStateOnStartupMetric(AuthStatus.EXPIRED, Constants.AWS_BUILDER_ID_URL)); + } + } + + @Test + void startupUserStateIsEmittedOnlyOncePerSession() { + when(pluginStore.get(Constants.LOGIN_TYPE_KEY)).thenReturn(LoginType.NONE.name()); + + try (MockedStatic mockedAuthTelemetryProvider = mockStatic(AuthTelemetryProvider.class)) { + DefaultAuthStateManager newManager = new DefaultAuthStateManager(pluginStore); + + newManager.toLoggedIn(LoginType.BUILDER_ID, loginParams, "ssoTokenId"); + newManager.toExpired(); + + mockedAuthTelemetryProvider.verify( + () -> AuthTelemetryProvider.emitUserStateOnStartupMetric(any(AuthStatus.class), any()), times(1)); + } + } + @Test void setAuthStateFieldsSuccess() { String ssoTokenId = "ssoTokenId"; diff --git a/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultLoginServiceTest.java b/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultLoginServiceTest.java index b2a95ddd1..4672b74de 100644 --- a/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultLoginServiceTest.java +++ b/plugin/tst/software/aws/toolkits/eclipse/amazonq/lsp/auth/DefaultLoginServiceTest.java @@ -3,23 +3,32 @@ package software.aws.toolkits.eclipse.amazonq.lsp.auth; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import java.time.Duration; +import java.time.Instant; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; import software.aws.toolkits.eclipse.amazonq.configuration.DefaultPluginStore; @@ -37,9 +46,13 @@ import software.aws.toolkits.eclipse.amazonq.lsp.model.UpdateCredentialsPayload; import software.aws.toolkits.eclipse.amazonq.plugin.Activator; import software.aws.toolkits.eclipse.amazonq.providers.lsp.LspProvider; +import software.aws.toolkits.eclipse.amazonq.telemetry.AwsTelemetryProvider; +import software.aws.toolkits.eclipse.amazonq.telemetry.AwsTelemetryProvider.BrowserLoginParams; import software.aws.toolkits.eclipse.amazonq.util.AuthUtil; import software.aws.toolkits.eclipse.amazonq.util.Constants; import software.aws.toolkits.eclipse.amazonq.util.LoggingService; +import software.aws.toolkits.telemetry.TelemetryDefinitions.CredentialType; +import software.aws.toolkits.telemetry.TelemetryDefinitions.Result; public final class DefaultLoginServiceTest { @@ -57,6 +70,7 @@ public final class DefaultLoginServiceTest { private static GetSsoTokenResult expectedSsoToken; private static SsoToken ssoToken; private static MockedStatic mockedCustomizationUtil; + private static MockedStatic mockedAwsTelemetryProvider; @BeforeEach public void setUp() { @@ -72,6 +86,7 @@ public void setUp() { mockedAuthUtil = mockStatic(AuthUtil.class); mockedActivator.when(Activator::getLspProvider).thenReturn(mockLspProvider); mockedCustomizationUtil = mockStatic(CustomizationUtil.class); + mockedAwsTelemetryProvider = mockStatic(AwsTelemetryProvider.class); updateCredentialsPayload = mock(UpdateCredentialsPayload.class); when(updateCredentialsPayload.data()).thenReturn("data"); @@ -99,6 +114,7 @@ void tearDown() throws Exception { mockedActivator.close(); mockedAuthUtil.close(); mockedCustomizationUtil.close(); + mockedAwsTelemetryProvider.close(); } @Test @@ -417,6 +433,163 @@ void processLoginIdcWithLoginOnInvalidTokenSuccess() throws Exception { verify(mockLoggingService).info("Successfully logged in"); } + @Test + void processLoginEmitsBrowserLoginSucceededWithoutSessionDurationOnFirstLogin() throws Exception { + LoginType loginType = LoginType.BUILDER_ID; + LoginParams loginParams = createLoginParams(createLoginIdcParams("test-region", "test-url")); + mockedAuthUtil.when(() -> AuthUtil.getIssuerUrl(loginType, loginParams)).thenReturn(Constants.AWS_BUILDER_ID_URL); + + when(mockedAuthTokenService.getSsoToken(loginType, loginParams, true)) + .thenReturn(CompletableFuture.completedFuture(expectedSsoToken)); + when(mockedAuthCredentialsService.updateTokenCredentials(expectedSsoToken.updateCredentialsParams())) + .thenReturn(CompletableFuture.completedFuture(null)); + + invokeProcessLogin(loginType, loginParams, true); + + BrowserLoginParams params = captureBrowserLoginParams(); + assertEquals(Constants.AWS_BUILDER_ID_URL, params.credentialStartUrl()); + assertEquals(CredentialType.BEARER_TOKEN, params.credentialType()); + assertEquals(Result.SUCCEEDED, params.result()); + assertFalse(params.isReAuth()); + assertNull(params.reason()); + assertNull(params.sessionDuration()); + } + + @Test + void processLoginRecordsLoginTimestampOnSuccess() throws Exception { + LoginType loginType = LoginType.BUILDER_ID; + LoginParams loginParams = createLoginParams(createLoginIdcParams("test-region", "test-url")); + mockedAuthUtil.when(() -> AuthUtil.getIssuerUrl(loginType, loginParams)).thenReturn(Constants.AWS_BUILDER_ID_URL); + + when(mockedAuthTokenService.getSsoToken(loginType, loginParams, true)) + .thenReturn(CompletableFuture.completedFuture(expectedSsoToken)); + when(mockedAuthCredentialsService.updateTokenCredentials(expectedSsoToken.updateCredentialsParams())) + .thenReturn(CompletableFuture.completedFuture(null)); + + invokeProcessLogin(loginType, loginParams, true); + + verify(mockPluginStore).put(Constants.LOGIN_TIMESTAMP_START_URL_KEY, Constants.AWS_BUILDER_ID_URL); + verify(mockPluginStore).put(eq(Constants.LOGIN_TIMESTAMP_KEY), any(String.class)); + } + + @Test + void processLoginEmitsBrowserLoginSucceededWithSessionDurationWhenPreviousLoginRecorded() throws Exception { + LoginType loginType = LoginType.BUILDER_ID; + LoginParams loginParams = createLoginParams(createLoginIdcParams("test-region", "test-url")); + mockedAuthUtil.when(() -> AuthUtil.getIssuerUrl(loginType, loginParams)).thenReturn(Constants.AWS_BUILDER_ID_URL); + long previousLoginMillis = Instant.now().minus(Duration.ofDays(30)).toEpochMilli(); + when(mockPluginStore.get(Constants.LOGIN_TIMESTAMP_START_URL_KEY)).thenReturn(Constants.AWS_BUILDER_ID_URL); + when(mockPluginStore.get(Constants.LOGIN_TIMESTAMP_KEY)).thenReturn(String.valueOf(previousLoginMillis)); + + when(mockedAuthTokenService.getSsoToken(loginType, loginParams, true)) + .thenReturn(CompletableFuture.completedFuture(expectedSsoToken)); + when(mockedAuthCredentialsService.updateTokenCredentials(expectedSsoToken.updateCredentialsParams())) + .thenReturn(CompletableFuture.completedFuture(null)); + + invokeProcessLogin(loginType, loginParams, true); + + BrowserLoginParams params = captureBrowserLoginParams(); + assertEquals(Result.SUCCEEDED, params.result()); + assertNotNull(params.sessionDuration()); + assertTrue(params.sessionDuration() >= Duration.ofDays(30).toMillis(), + "session duration should cover the recorded previous login"); + } + + @Test + void processLoginEmitsBrowserLoginWithIsReAuthWhenReAuthenticating() throws Exception { + LoginType loginType = LoginType.BUILDER_ID; + LoginParams loginParams = createLoginParams(createLoginIdcParams("test-region", "test-url")); + mockedAuthUtil.when(() -> AuthUtil.getIssuerUrl(loginType, loginParams)).thenReturn(Constants.AWS_BUILDER_ID_URL); + + when(mockedAuthTokenService.getSsoToken(loginType, loginParams, true)) + .thenReturn(CompletableFuture.completedFuture(expectedSsoToken)); + when(mockedAuthCredentialsService.updateTokenCredentials(expectedSsoToken.updateCredentialsParams())) + .thenReturn(CompletableFuture.completedFuture(null)); + + invokeProcessLogin(loginType, loginParams, true, true); + + BrowserLoginParams params = captureBrowserLoginParams(); + assertTrue(params.isReAuth()); + assertEquals(Result.SUCCEEDED, params.result()); + } + + @Test + void processLoginDoesNotEmitBrowserLoginOnSilentTokenRefresh() throws Exception { + // The re-authentication performed on start up passes loginOnInvalidToken=false: the cached token + // is refreshed without opening the browser, so it is neither a browser login nor a new session. + LoginType loginType = LoginType.BUILDER_ID; + LoginParams loginParams = createLoginParams(createLoginIdcParams("test-region", "test-url")); + mockedAuthUtil.when(() -> AuthUtil.getIssuerUrl(loginType, loginParams)).thenReturn(Constants.AWS_BUILDER_ID_URL); + + when(mockedAuthTokenService.getSsoToken(loginType, loginParams, false)) + .thenReturn(CompletableFuture.completedFuture(expectedSsoToken)); + when(mockedAuthCredentialsService.updateTokenCredentials(expectedSsoToken.updateCredentialsParams())) + .thenReturn(CompletableFuture.completedFuture(null)); + + invokeProcessLogin(loginType, loginParams, false, true); + + mockedAwsTelemetryProvider.verifyNoInteractions(); + verify(mockPluginStore, never()).put(eq(Constants.LOGIN_TIMESTAMP_KEY), any(String.class)); + } + + @Test + void processLoginDoesNotEmitBrowserLoginFailedOnSilentTokenRefresh() { + LoginType loginType = LoginType.BUILDER_ID; + LoginParams loginParams = createLoginParams(createLoginIdcParams("test-region", "test-url")); + mockedAuthUtil.when(() -> AuthUtil.getIssuerUrl(loginType, loginParams)).thenReturn(Constants.AWS_BUILDER_ID_URL); + + when(mockedAuthTokenService.getSsoToken(loginType, loginParams, false)) + .thenReturn(CompletableFuture.failedFuture(new IllegalStateException("token unavailable"))); + + CompletableFuture result = loginService.processLogin(loginType, loginParams, false, true); + + assertThrows(ExecutionException.class, result::get); + mockedAwsTelemetryProvider.verifyNoInteractions(); + } + + @Test + void processLoginEmitsBrowserLoginFailedWithExceptionReason() { + LoginType loginType = LoginType.BUILDER_ID; + LoginParams loginParams = createLoginParams(createLoginIdcParams("test-region", "test-url")); + mockedAuthUtil.when(() -> AuthUtil.getIssuerUrl(loginType, loginParams)).thenReturn(Constants.AWS_BUILDER_ID_URL); + + when(mockedAuthTokenService.getSsoToken(loginType, loginParams, true)) + .thenReturn(CompletableFuture.failedFuture(new IllegalStateException("token unavailable"))); + + CompletableFuture result = loginService.processLogin(loginType, loginParams, true, false); + + assertThrows(ExecutionException.class, result::get); + + BrowserLoginParams params = captureBrowserLoginParams(); + assertEquals(Constants.AWS_BUILDER_ID_URL, params.credentialStartUrl()); + assertEquals(Result.FAILED, params.result()); + assertEquals("IllegalStateException", params.reason()); + assertNull(params.sessionDuration()); + verify(mockPluginStore, never()).put(eq(Constants.LOGIN_TIMESTAMP_KEY), any(String.class)); + } + + @Test + void processLoginDoesNotEmitBrowserLoginWhenStartUrlIsUnknown() throws Exception { + LoginType loginType = LoginType.BUILDER_ID; + LoginParams loginParams = createLoginParams(createLoginIdcParams("test-region", "test-url")); + mockedAuthUtil.when(() -> AuthUtil.getIssuerUrl(loginType, loginParams)).thenReturn(null); + + when(mockedAuthTokenService.getSsoToken(loginType, loginParams, true)) + .thenReturn(CompletableFuture.completedFuture(expectedSsoToken)); + when(mockedAuthCredentialsService.updateTokenCredentials(expectedSsoToken.updateCredentialsParams())) + .thenReturn(CompletableFuture.completedFuture(null)); + + invokeProcessLogin(loginType, loginParams, true); + + mockedAwsTelemetryProvider.verifyNoInteractions(); + } + + private BrowserLoginParams captureBrowserLoginParams() { + ArgumentCaptor captor = ArgumentCaptor.forClass(BrowserLoginParams.class); + mockedAwsTelemetryProvider.verify(() -> AwsTelemetryProvider.emitLoginWithBrowserEvent(captor.capture())); + return captor.getValue(); + } + private LoginParams createLoginParams(final LoginIdcParams idcParams) { LoginParams loginParams = mock(LoginParams.class); when(loginParams.getLoginIdcParams()).thenReturn(idcParams); @@ -459,7 +632,12 @@ private AuthState createAuthState(final AuthStateType authStateType, final Login private void invokeProcessLogin(final LoginType loginType, final LoginParams loginParams, final boolean loginOnInvalidToken) throws Exception { - Object processLoginFuture = loginService.processLogin(loginType, loginParams, loginOnInvalidToken); + invokeProcessLogin(loginType, loginParams, loginOnInvalidToken, false); + } + + private void invokeProcessLogin(final LoginType loginType, final LoginParams loginParams, + final boolean loginOnInvalidToken, final boolean isReAuth) throws Exception { + Object processLoginFuture = loginService.processLogin(loginType, loginParams, loginOnInvalidToken, isReAuth); assertTrue(processLoginFuture instanceof CompletableFuture, "Return value should be CompletableFuture"); CompletableFuture future = (CompletableFuture) processLoginFuture; diff --git a/plugin/tst/software/aws/toolkits/eclipse/amazonq/providers/assets/ToolkitLoginWebViewAssetProviderTest.java b/plugin/tst/software/aws/toolkits/eclipse/amazonq/providers/assets/ToolkitLoginWebViewAssetProviderTest.java new file mode 100644 index 000000000..99e6d0b05 --- /dev/null +++ b/plugin/tst/software/aws/toolkits/eclipse/amazonq/providers/assets/ToolkitLoginWebViewAssetProviderTest.java @@ -0,0 +1,75 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.aws.toolkits.eclipse.amazonq.providers.assets; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mockConstruction; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.net.URL; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; + +import software.aws.toolkits.eclipse.amazonq.extensions.implementation.ActivatorStaticMockExtension; +import software.aws.toolkits.eclipse.amazonq.telemetry.ToolkitTelemetryProvider; +import software.aws.toolkits.eclipse.amazonq.util.PluginUtils; +import software.aws.toolkits.eclipse.amazonq.util.ThreadingUtils; +import software.aws.toolkits.eclipse.amazonq.util.WebviewAssetServer; +import software.aws.toolkits.telemetry.TelemetryDefinitions.Result; + +public final class ToolkitLoginWebViewAssetProviderTest { + + @RegisterExtension + private static ActivatorStaticMockExtension activatorStaticMockExtension = new ActivatorStaticMockExtension(); + + @Test + void initializeEmitsDidLoadLoginModuleFailedWhenAssetsAreMissing() { + try (MockedStatic mockedThreadingUtils = mockStatic(ThreadingUtils.class); + MockedStatic mockedPluginUtils = mockStatic(PluginUtils.class); + MockedStatic mockedToolkitTelemetryProvider = mockStatic(ToolkitTelemetryProvider.class)) { + mockedPluginUtils.when(() -> PluginUtils.getResource("webview/build/assets/js/getStart.js")) + .thenThrow(new IOException("resource is unavailable")); + + ToolkitLoginWebViewAssetProvider assetProvider = new ToolkitLoginWebViewAssetProvider(); + assetProvider.initialize(); + + ArgumentCaptor captor = ArgumentCaptor.forClass(Runnable.class); + mockedThreadingUtils.verify(() -> ThreadingUtils.executeAsyncTask(captor.capture())); + captor.getValue().run(); + + mockedToolkitTelemetryProvider.verify(() -> ToolkitTelemetryProvider.emitDidLoadModuleEventMetric( + ToolkitTelemetryProvider.LOGIN_MODULE, Result.FAILED, "DependencyMissing"), times(1)); + } + } + + @Test + void initializeEmitsDidLoadLoginModuleFailedWhenAssetsCannotBeServed() throws Exception { + try (MockedStatic mockedThreadingUtils = mockStatic(ThreadingUtils.class); + MockedStatic mockedPluginUtils = mockStatic(PluginUtils.class); + MockedConstruction mockedWebviewAssetServer = mockConstruction(WebviewAssetServer.class, + (mock, context) -> when(mock.resolve(any(String.class))).thenReturn(false)); + MockedStatic mockedToolkitTelemetryProvider = mockStatic(ToolkitTelemetryProvider.class)) { + mockedPluginUtils.when(() -> PluginUtils.getResource("webview/build/assets/js/getStart.js")) + .thenReturn(new URL("file:/assets/js/getStart.js")); + + ToolkitLoginWebViewAssetProvider assetProvider = new ToolkitLoginWebViewAssetProvider(); + assetProvider.initialize(); + + ArgumentCaptor captor = ArgumentCaptor.forClass(Runnable.class); + mockedThreadingUtils.verify(() -> ThreadingUtils.executeAsyncTask(captor.capture())); + captor.getValue().run(); + + mockedToolkitTelemetryProvider.verify(() -> ToolkitTelemetryProvider.emitDidLoadModuleEventMetric( + ToolkitTelemetryProvider.LOGIN_MODULE, Result.FAILED, "AssetLoadFailed"), times(1)); + } + } + +} diff --git a/plugin/tst/software/aws/toolkits/eclipse/amazonq/telemetry/AwsTelemetryProviderTest.java b/plugin/tst/software/aws/toolkits/eclipse/amazonq/telemetry/AwsTelemetryProviderTest.java new file mode 100644 index 000000000..03ab906af --- /dev/null +++ b/plugin/tst/software/aws/toolkits/eclipse/amazonq/telemetry/AwsTelemetryProviderTest.java @@ -0,0 +1,86 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.aws.toolkits.eclipse.amazonq.telemetry; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.verify; + +import java.util.Optional; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.mockito.ArgumentCaptor; + +import software.amazon.awssdk.services.toolkittelemetry.model.MetadataEntry; +import software.amazon.awssdk.services.toolkittelemetry.model.MetricDatum; +import software.aws.toolkits.eclipse.amazonq.extensions.implementation.ActivatorStaticMockExtension; +import software.aws.toolkits.eclipse.amazonq.telemetry.AwsTelemetryProvider.BrowserLoginParams; +import software.aws.toolkits.eclipse.amazonq.telemetry.service.TelemetryService; +import software.aws.toolkits.eclipse.amazonq.util.Constants; +import software.aws.toolkits.telemetry.TelemetryDefinitions.CredentialType; +import software.aws.toolkits.telemetry.TelemetryDefinitions.Result; + +public final class AwsTelemetryProviderTest { + + private static final String SESSION_DURATION_KEY = "sessionDuration"; + + @RegisterExtension + private static ActivatorStaticMockExtension activatorStaticMockExtension = new ActivatorStaticMockExtension(); + + @Test + void emitLoginWithBrowserEventReportsSessionDurationWhenKnown() { + AwsTelemetryProvider.emitLoginWithBrowserEvent(new BrowserLoginParams(Constants.AWS_BUILDER_ID_URL, + CredentialType.BEARER_TOKEN, false, Result.SUCCEEDED, null, 7776000000L)); + + MetricDatum metricDatum = captureMetricDatum(); + assertEquals("aws_loginWithBrowser", metricDatum.metricName()); + assertEquals(Optional.of("7776000000"), getMetadataValue(metricDatum, SESSION_DURATION_KEY)); + assertEquals(Optional.of(Constants.AWS_BUILDER_ID_URL), getMetadataValue(metricDatum, "credentialStartUrl")); + assertEquals(Optional.of("bearerToken"), getMetadataValue(metricDatum, "credentialType")); + assertEquals(Optional.of("Succeeded"), getMetadataValue(metricDatum, "result")); + assertEquals(Optional.of("false"), getMetadataValue(metricDatum, "isReAuth")); + assertFalse(metricDatum.passive()); + } + + @Test + void emitLoginWithBrowserEventLeavesOutSessionDurationWhenUnknown() { + AwsTelemetryProvider.emitLoginWithBrowserEvent(new BrowserLoginParams(Constants.AWS_BUILDER_ID_URL, + CredentialType.BEARER_TOKEN, false, Result.SUCCEEDED, null, null)); + + MetricDatum metricDatum = captureMetricDatum(); + assertTrue(getMetadataValue(metricDatum, SESSION_DURATION_KEY).isEmpty(), + "session duration should not be reported when no previous login is known"); + assertEquals(Optional.of(Constants.AWS_BUILDER_ID_URL), getMetadataValue(metricDatum, "credentialStartUrl")); + } + + @Test + void emitLoginWithBrowserEventReportsReasonCodeOnFailure() { + AwsTelemetryProvider.emitLoginWithBrowserEvent(new BrowserLoginParams(Constants.AWS_BUILDER_ID_URL, + CredentialType.BEARER_TOKEN, true, Result.FAILED, "IllegalStateException", null)); + + MetricDatum metricDatum = captureMetricDatum(); + assertEquals(Optional.of("Failed"), getMetadataValue(metricDatum, "result")); + assertEquals(Optional.of("IllegalStateException"), getMetadataValue(metricDatum, "reason")); + assertEquals(Optional.of("true"), getMetadataValue(metricDatum, "isReAuth")); + assertTrue(getMetadataValue(metricDatum, SESSION_DURATION_KEY).isEmpty(), + "session duration should not be reported for a failed login"); + } + + private MetricDatum captureMetricDatum() { + TelemetryService telemetryService = activatorStaticMockExtension.getMock(TelemetryService.class); + ArgumentCaptor captor = ArgumentCaptor.forClass(MetricDatum.class); + verify(telemetryService).emitMetric(captor.capture()); + return captor.getValue(); + } + + private Optional getMetadataValue(final MetricDatum metricDatum, final String key) { + return metricDatum.metadata().stream() + .filter(entry -> key.equals(entry.key())) + .map(MetadataEntry::value) + .findFirst(); + } + +} diff --git a/plugin/tst/software/aws/toolkits/eclipse/amazonq/telemetry/ToolkitTelemetryProviderTest.java b/plugin/tst/software/aws/toolkits/eclipse/amazonq/telemetry/ToolkitTelemetryProviderTest.java new file mode 100644 index 000000000..6656acbf5 --- /dev/null +++ b/plugin/tst/software/aws/toolkits/eclipse/amazonq/telemetry/ToolkitTelemetryProviderTest.java @@ -0,0 +1,63 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.aws.toolkits.eclipse.amazonq.telemetry; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.verify; + +import java.util.Optional; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.mockito.ArgumentCaptor; + +import software.amazon.awssdk.services.toolkittelemetry.model.MetadataEntry; +import software.amazon.awssdk.services.toolkittelemetry.model.MetricDatum; +import software.aws.toolkits.eclipse.amazonq.extensions.implementation.ActivatorStaticMockExtension; +import software.aws.toolkits.eclipse.amazonq.telemetry.service.TelemetryService; +import software.aws.toolkits.telemetry.TelemetryDefinitions.Result; + +public final class ToolkitTelemetryProviderTest { + + @RegisterExtension + private static ActivatorStaticMockExtension activatorStaticMockExtension = new ActivatorStaticMockExtension(); + + @Test + void emitDidLoadModuleEventMetricReportsLoginModuleLoaded() { + ToolkitTelemetryProvider.emitDidLoadModuleEventMetric(ToolkitTelemetryProvider.LOGIN_MODULE, Result.SUCCEEDED, null); + + MetricDatum metricDatum = captureMetricDatum(); + assertEquals("toolkit_didLoadModule", metricDatum.metricName()); + assertEquals(Optional.of("login"), getMetadataValue(metricDatum, "module")); + assertEquals(Optional.of("Succeeded"), getMetadataValue(metricDatum, "result")); + assertTrue(metricDatum.passive()); + } + + @Test + void emitDidLoadModuleEventMetricReportsReasonCodeOnFailure() { + ToolkitTelemetryProvider.emitDidLoadModuleEventMetric(ToolkitTelemetryProvider.LOGIN_MODULE, Result.FAILED, + "DependencyMissing"); + + MetricDatum metricDatum = captureMetricDatum(); + assertEquals(Optional.of("login"), getMetadataValue(metricDatum, "module")); + assertEquals(Optional.of("Failed"), getMetadataValue(metricDatum, "result")); + assertEquals(Optional.of("DependencyMissing"), getMetadataValue(metricDatum, "reason")); + } + + private MetricDatum captureMetricDatum() { + TelemetryService telemetryService = activatorStaticMockExtension.getMock(TelemetryService.class); + ArgumentCaptor captor = ArgumentCaptor.forClass(MetricDatum.class); + verify(telemetryService).emitMetric(captor.capture()); + return captor.getValue(); + } + + private Optional getMetadataValue(final MetricDatum metricDatum, final String key) { + return metricDatum.metadata().stream() + .filter(entry -> key.equals(entry.key())) + .map(MetadataEntry::value) + .findFirst(); + } + +}