From f21bbcb1b84bbf10fdd809d3f204bf88dbb51c56 Mon Sep 17 00:00:00 2001 From: Lukas Boehler Date: Wed, 30 Sep 2026 21:42:01 -0400 Subject: [PATCH 01/25] fix: take screenshots on Android 7.1 with hardware acceleration API 25 went to generateBitmap, which skips hardware accelerated windows (all of them by default), so hardware accelerated apps on Android 7.1 got no screenshot. Below API 26 the activity window is now drawn into the bitmap, as on API 24 and lower. Co-Authored-By: Claude Opus 5.5 --- gleap/src/main/java/io/gleap/ScreenshotUtil.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gleap/src/main/java/io/gleap/ScreenshotUtil.java b/gleap/src/main/java/io/gleap/ScreenshotUtil.java index d5c4b1e2..01cf5b0a 100644 --- a/gleap/src/main/java/io/gleap/ScreenshotUtil.java +++ b/gleap/src/main/java/io/gleap/ScreenshotUtil.java @@ -84,7 +84,10 @@ public void invoke(Bitmap bitmap) { getImageCallback.getImage(bitmap != null ? getResizedBitmap(bitmap) : null); } }); - } else if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.N) { + } else if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) { + // Below API 26 there is no PixelCopy of a window: draw it. API 25 went to + // generateBitmap, which skips hardware accelerated windows (all of them by default), + // so hardware accelerated apps got no screenshot at all. bitmap = Bitmap.createBitmap(view.getWidth(), view.getHeight(), Bitmap.Config.ARGB_8888); Canvas canvas = new Canvas(bitmap); From 075314af9efe7faf592a896711a205c14887779d Mon Sep 17 00:00:00 2001 From: Lukas Boehler Date: Wed, 30 Sep 2026 21:42:01 -0400 Subject: [PATCH 02/25] fix: copy attachment uploads in chunks instead of whole files addFilePart read each file into one array sized to the whole file (and a single read() could return less than the file). It now copies the file in 64 KB chunks; a file that cannot be read still leaves its part empty, and errors writing the request still fail the upload. Co-Authored-By: Claude Opus 5.5 --- .../java/io/gleap/FormDataHttpsHelper.java | 33 ++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/gleap/src/main/java/io/gleap/FormDataHttpsHelper.java b/gleap/src/main/java/io/gleap/FormDataHttpsHelper.java index 914551ee..2806f517 100644 --- a/gleap/src/main/java/io/gleap/FormDataHttpsHelper.java +++ b/gleap/src/main/java/io/gleap/FormDataHttpsHelper.java @@ -60,14 +60,37 @@ public void addFilePart(File uploadFile) fileName + "\"" + this.crlf); request.writeBytes("Content-Type: " + "" + URLConnection.guessContentTypeFromName(fileName) + this.crlf + this.crlf); - int size = uploadFile != null ? (int) uploadFile.length() : 0; - byte[] bytes = new byte[size]; - try (BufferedInputStream buf = new BufferedInputStream(new FileInputStream(uploadFile))) { - buf.read(bytes, 0, bytes.length); + // Copied in chunks: the file is never held in memory as a whole (and a single read could + // return less than the whole file). A file that cannot be read leaves the part empty. + InputStream file = null; + try { + file = new BufferedInputStream(new FileInputStream(uploadFile)); } catch (IOException e) { GleapLog.w("Could not read the attachment " + fileName, e); } - request.write(bytes); + if (file != null) { + try { + byte[] chunk = new byte[64 * 1024]; + while (true) { + int read; + try { + read = file.read(chunk); + } catch (IOException e) { + GleapLog.w("Could not read the attachment " + fileName, e); + break; + } + if (read == -1) { + break; + } + request.write(chunk, 0, read); + } + } finally { + try { + file.close(); + } catch (IOException ignore) { + } + } + } request.writeBytes(this.crlf); } From a1476302afe6f828442f521b709fdba0d7c53884 Mon Sep 17 00:00:00 2001 From: Lukas Boehler Date: Wed, 30 Sep 2026 21:42:24 -0400 Subject: [PATCH 03/25] feat: background log collection for capture requests When the server asks for the app's logs (a capture request of kind "logs", pushed on the WebSocket as capture-request in any widget state, or returned as "cr" with a ping), the SDK claims the request, gives a wrapper SDK up to 500 ms to hand over its buffered logs (setLogFlushHandler), and sends what a ticket carries (console log incl. the logcat snapshot, network logs, custom data, env data, events; the replay only when requested and enabled, its frames uploaded through /uploads/sdksteps first) gzip compressed to POST /v3/shared/capture-requests/{id}/logs. Each request id is sent once per process; a request that failed on the server side is tried again when it is pushed again. With setRemoteLogCollectionEnabled(false) such requests are answered with an "unsupported" event and nothing is collected. The SDK announces what it can capture: caps on the WebSocket url and in the ping body. Replay frames are no longer recycled when they drop out of the replay: an upload may still be reading them. Nothing runs before a request arrives; the work runs on two lazily started daemon threads that end when idle. Co-Authored-By: Claude Opus 5.5 --- .../main/java/io/gleap/FeedbackUploader.java | 47 ++ gleap/src/main/java/io/gleap/Gleap.java | 26 + .../src/main/java/io/gleap/GleapCapture.java | 308 ++++++++++++ .../main/java/io/gleap/GleapCaptureApi.java | 301 +++++++++++ .../java/io/gleap/GleapCaptureExecutor.java | 45 ++ .../main/java/io/gleap/GleapCaptureLogs.java | 469 ++++++++++++++++++ .../main/java/io/gleap/GleapEventService.java | 46 +- .../java/io/gleap/GleapLogFlushHandler.java | 19 + .../java/io/gleap/GleapWebSocketListener.java | 12 +- gleap/src/main/java/io/gleap/Replay.java | 3 +- .../java/io/gleap/GleapCaptureLogsTest.java | 328 ++++++++++++ 11 files changed, 1601 insertions(+), 3 deletions(-) create mode 100644 gleap/src/main/java/io/gleap/GleapCapture.java create mode 100644 gleap/src/main/java/io/gleap/GleapCaptureApi.java create mode 100644 gleap/src/main/java/io/gleap/GleapCaptureExecutor.java create mode 100644 gleap/src/main/java/io/gleap/GleapCaptureLogs.java create mode 100644 gleap/src/main/java/io/gleap/GleapLogFlushHandler.java create mode 100644 gleap/src/test/java/io/gleap/GleapCaptureLogsTest.java diff --git a/gleap/src/main/java/io/gleap/FeedbackUploader.java b/gleap/src/main/java/io/gleap/FeedbackUploader.java index f79b0837..4da50140 100644 --- a/gleap/src/main/java/io/gleap/FeedbackUploader.java +++ b/gleap/src/main/java/io/gleap/FeedbackUploader.java @@ -157,6 +157,53 @@ private JSONArray uploadReplayFrames() throws IOException, JSONException { return result; } + /** + * Uploads these replay frames (/uploads/sdksteps) for the logs of a capture request and + * returns their entries, oldest first. The replay itself is not cleared. Frames that cannot + * be written are left out, and no request is made without frames. + */ + JSONArray uploadReplayFrames(ScreenshotReplay[] replays) throws IOException, JSONException { + JSONArray result = new JSONArray(); + List included = new LinkedList<>(); + List files = new LinkedList<>(); + try { + for (ScreenshotReplay replay : replays) { + if (replay == null || replay.getScreenshot() == null) { + continue; + } + File file = bitmapToFile(replay.getScreenshot()); + if (file != null) { + files.add(file); + included.add(replay); + } + } + if (files.isEmpty()) { + return result; + } + + GleapConfig config = GleapConfig.getInstance(); + FormDataHttpsHelper multipart = new FormDataHttpsHelper(config.getApiUrl() + UPLOAD_IMAGE_MULTI_BACKEND_URL_POSTFIX, config.getSdkKey()); + for (File file : files) { + multipart.addFilePart(file); + } + JSONArray fileUrls = new JSONObject(multipart.finishAndUpload()).getJSONArray("fileUrls"); + for (int i = 0; i < fileUrls.length() && i < included.size(); i++) { + ScreenshotReplay replay = included.get(i); + JSONObject entry = new JSONObject(); + entry.put("url", fileUrls.get(i)); + entry.put("screenname", replay.getScreenName()); + entry.put("date", DateUtil.dateToString(replay.getDate())); + entry.put("interactions", new JSONArray()); + result.put(entry); + } + } finally { + for (File file : files) { + delete(file); + } + } + return result; + } + // The PNGs are only needed for the upload: screenshots of the app must not stay on disk. private static void delete(File file) { if (file != null && file.exists() && !file.delete()) { diff --git a/gleap/src/main/java/io/gleap/Gleap.java b/gleap/src/main/java/io/gleap/Gleap.java index b287e6c0..e2bec8db 100644 --- a/gleap/src/main/java/io/gleap/Gleap.java +++ b/gleap/src/main/java/io/gleap/Gleap.java @@ -933,6 +933,32 @@ public void setBitmapCallback(GetBitmapCallback getBitmapCallback) { GleapErrors.guard("setBitmapCallback", () -> GleapCallbacks.getInstance().setGetBitmapCallback(getBitmapCallback)); } + /** + * Enables or disables sending the app's logs for capture requests: a workflow or an AI agent + * can ask for the logs of the app while it runs (no customer action), and screenshots and + * recordings can bring the logs around them. The logs are what a ticket carries (console and + * network logs, custom data, env data, events; the replay only when it is asked for and + * replays are enabled) and the existing settings apply (e.g. {@link #disableConsoleLog()}, + * {@link #setDisableEnvData(boolean)}). While disabled, log requests are answered as + * unsupported and captures are sent without logs. Enabled by default. + * + * @param enabled false to never send logs for capture requests + */ + public void setRemoteLogCollectionEnabled(boolean enabled) { + GleapErrors.guard("setRemoteLogCollectionEnabled", () -> GleapCapture.setRemoteLogCollectionEnabled(enabled)); + } + + /** + * For wrapper SDKs (React Native, Flutter, Capacitor): called right before the logs for a + * capture request are collected, so the wrapper can hand over the logs it buffers itself + * (see {@link GleapLogFlushHandler}). The SDK waits at most 500 ms for it. null removes it. + * + * @param handler called on the main thread before the logs are collected + */ + public void setLogFlushHandler(GleapLogFlushHandler handler) { + GleapErrors.guard("setLogFlushHandler", () -> GleapCapture.setLogFlushHandler(handler)); + } + /** * This is called, when the config is received from the server. The config is loaded once per * process: a callback set after it was loaded is called once with the loaded config, posted diff --git a/gleap/src/main/java/io/gleap/GleapCapture.java b/gleap/src/main/java/io/gleap/GleapCapture.java new file mode 100644 index 00000000..875b6ba8 --- /dev/null +++ b/gleap/src/main/java/io/gleap/GleapCapture.java @@ -0,0 +1,308 @@ +package io.gleap; + +import android.app.Application; +import android.content.Context; +import android.content.SharedPreferences; +import android.os.Build; +import android.view.View; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.File; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.WeakHashMap; +import java.util.regex.Pattern; + +import gleap.io.gleap.BuildConfig; + +/** + * Capture requests ("show me the issue"): the app's switches for them, the views it masks, and + * what the SDK reports it can do (to the widget, on the WebSocket and with the pings). + *

+ * Nothing here runs on its own: the capture code only starts when a request arrives, from the + * widget ({@link GleapCaptureCoordinator}) or from the server ({@link GleapCaptureLogs}). + */ +final class GleapCapture { + static final int CAPABILITIES_VERSION = 1; + static final String PLATFORM = "android"; + // Native in-app frame capture (screenshots and recordings alike). + static final String METHOD_FRAMES = "frames"; + + static final String CAP_SCREENSHOT = "capture.screenshot"; + static final String CAP_RECORDING = "capture.recording"; + static final String CAP_LOGS = "capture.logs"; + + static final int MIN_RECORDING_SEC = 5; + static final int DEFAULT_RECORDING_SEC = 60; + static final int MAX_RECORDING_SEC = 180; + + // Server ids are ObjectIds; anything else is never put into a request path. + private static final Pattern REQUEST_ID = Pattern.compile("^[A-Za-z0-9_-]{1,64}$"); + private static final String PREFS = "gleap-capture"; + private static final String DEVICE_ID_KEY = "deviceId"; + private static final String CAPTURE_DIR = "gleap-capture"; + + private static volatile boolean captureEnabled = true; + private static volatile boolean remoteLogCollectionEnabled = true; + private static volatile GleapLogFlushHandler logFlushHandler; + // The views the app masked; weak, so a masked view never outlives its screen. Guarded by itself. + private static final Map maskedViews = new WeakHashMap<>(); + private static String deviceId; + + private GleapCapture() { + } + + static boolean isCaptureEnabled() { + return captureEnabled; + } + + static void setCaptureEnabled(boolean enabled) { + captureEnabled = enabled; + } + + static boolean isRemoteLogCollectionEnabled() { + return remoteLogCollectionEnabled; + } + + static void setRemoteLogCollectionEnabled(boolean enabled) { + remoteLogCollectionEnabled = enabled; + } + + static GleapLogFlushHandler getLogFlushHandler() { + return logFlushHandler; + } + + static void setLogFlushHandler(GleapLogFlushHandler handler) { + logFlushHandler = handler; + } + + static void maskView(View view) { + if (view == null) { + return; + } + synchronized (maskedViews) { + maskedViews.put(view, Boolean.TRUE); + } + } + + static void unmaskView(View view) { + if (view == null) { + return; + } + synchronized (maskedViews) { + maskedViews.remove(view); + } + } + + /** + * The masked views that are still alive, as an identity set. + */ + static Set maskedViews() { + synchronized (maskedViews) { + if (maskedViews.isEmpty()) { + return Collections.emptySet(); + } + Set views = Collections.newSetFromMap(new IdentityHashMap()); + views.addAll(maskedViews.keySet()); + views.remove(null); + return views; + } + } + + /** + * Recordings are frames of the app's own windows, encoded with MediaCodec: PixelCopy of a + * window needs API 26. + */ + static boolean isRecordingSupported() { + return Build.VERSION.SDK_INT >= Build.VERSION_CODES.O; + } + + static String sdkType() { + try { + APPLICATIONTYPE type = GleapBug.getInstance().getApplicationType(); + return type != null ? type.name() : APPLICATIONTYPE.NATIVE.name(); + } catch (Throwable ignore) { + return APPLICATIONTYPE.NATIVE.name(); + } + } + + static String sdkVersion() { + return BuildConfig.VERSION_NAME; + } + + /** + * The capability flags the SDK announces on the WebSocket and with the pings. + */ + static List caps(boolean captureEnabled, boolean recordingSupported, boolean logsEnabled) { + List caps = new ArrayList<>(); + if (captureEnabled) { + caps.add(CAP_SCREENSHOT); + if (recordingSupported) { + caps.add(CAP_RECORDING); + } + } + if (logsEnabled) { + caps.add(CAP_LOGS); + } + return caps; + } + + static List currentCaps() { + return caps(captureEnabled, isRecordingSupported(), remoteLogCollectionEnabled); + } + + static JSONArray currentCapsJson() { + JSONArray caps = new JSONArray(); + for (String cap : currentCaps()) { + caps.put(cap); + } + return caps; + } + + /** + * The value of the WebSocket url's {@code caps} parameter, e.g. + * {@code capture.screenshot,capture.recording,capture.logs}; empty when nothing is supported. + */ + static String capsQueryValue(List caps) { + StringBuilder value = new StringBuilder(); + for (String cap : caps) { + if (value.length() > 0) { + value.append(','); + } + value.append(cap); + } + return value.toString(); + } + + /** + * The capture-capabilities message data for the widget. With capture disabled nothing is + * offered, so the widget only offers the upload. + */ + static JSONObject capabilities(boolean captureEnabled, boolean recordingSupported, String sdkType, + String sdkVersion) throws JSONException { + JSONObject data = new JSONObject(); + data.put("version", CAPABILITIES_VERSION); + data.put("platform", PLATFORM); + data.put("sdkType", sdkType); + data.put("sdkVersion", sdkVersion); + data.put("screenshot", captureEnabled); + boolean recording = captureEnabled && recordingSupported; + data.put("recording", recording); + if (recording) { + data.put("recordingMethod", METHOD_FRAMES); + data.put("maxRecordingSec", MAX_RECORDING_SEC); + } + data.put("annotate", true); + data.put("microphone", false); + return data; + } + + static JSONObject currentCapabilities() throws JSONException { + return capabilities(captureEnabled, isRecordingSupported(), sdkType(), sdkVersion()); + } + + static boolean isValidRequestId(String id) { + return id != null && REQUEST_ID.matcher(id).matches(); + } + + /** + * Whether {@code url} has the scheme, host and port of {@code expectedUrl} (the widget url). + * Used to accept capture commands only from the Gleap messenger page. + */ + static boolean sameOrigin(String url, String expectedUrl) { + if (url == null || expectedUrl == null) { + return false; + } + try { + URL actual = new URL(url.trim()); + URL expected = new URL(expectedUrl.trim()); + String scheme = actual.getProtocol().toLowerCase(Locale.ROOT); + if (!scheme.equals("https") && !scheme.equals("http")) { + return false; + } + if (!scheme.equals(expected.getProtocol().toLowerCase(Locale.ROOT))) { + return false; + } + String host = actual.getHost(); + if (host == null || host.isEmpty() || !host.equalsIgnoreCase(expected.getHost())) { + return false; + } + return port(actual) == port(expected); + } catch (Exception e) { + return false; + } + } + + private static int port(URL url) { + return url.getPort() != -1 ? url.getPort() : url.getDefaultPort(); + } + + /** + * A random id of this installation, sent when the SDK claims a capture request (so the + * server can tell devices apart). Created on the first capture request. + */ + static synchronized String deviceId() { + if (deviceId != null) { + return deviceId; + } + try { + Application application = GleapInitializer.getApplication(); + if (application != null) { + SharedPreferences prefs = application.getSharedPreferences(PREFS, Context.MODE_PRIVATE); + String stored = prefs.getString(DEVICE_ID_KEY, null); + if (stored == null || stored.isEmpty()) { + stored = UUID.randomUUID().toString(); + prefs.edit().putString(DEVICE_ID_KEY, stored).apply(); + } + deviceId = stored; + return deviceId; + } + } catch (Throwable ignore) { + } + deviceId = UUID.randomUUID().toString(); + return deviceId; + } + + /** + * Where recordings are written while they are recorded, previewed and uploaded. + */ + static File captureDir(Context context) { + File dir = new File(context.getCacheDir(), CAPTURE_DIR); + if (!dir.exists() && !dir.mkdirs()) { + GleapLog.w("Could not create the capture directory"); + } + return dir; + } + + /** + * Deletes recordings left behind, e.g. by a process that died while it was recording. + */ + static void deleteCaptureFiles(Context context) { + try { + if (context == null) { + return; + } + File dir = new File(context.getCacheDir(), CAPTURE_DIR); + File[] files = dir.listFiles(); + if (files == null) { + return; + } + for (File file : files) { + if (!file.delete()) { + GleapLog.w("Could not delete " + file.getName()); + } + } + } catch (Throwable ignore) { + } + } +} diff --git a/gleap/src/main/java/io/gleap/GleapCaptureApi.java b/gleap/src/main/java/io/gleap/GleapCaptureApi.java new file mode 100644 index 00000000..197d8b5e --- /dev/null +++ b/gleap/src/main/java/io/gleap/GleapCaptureApi.java @@ -0,0 +1,301 @@ +package io.gleap; + +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import okhttp3.Call; +import okhttp3.MediaType; +import okhttp3.MultipartBody; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.ResponseBody; +import okio.Buffer; +import okio.BufferedSink; +import okio.Okio; +import okio.Source; + +/** + * The capture request endpoints the SDK calls (shared routes: the SDK key and the session), and + * the streaming upload of recordings. All calls block: run them on a background thread. + */ +final class GleapCaptureApi { + static final String PATH_PREFIX = "/v3/shared/capture-requests/"; + static final String UPLOAD_PATH = "/uploads/attachments"; + // The server accepts log bundles up to 20 MB. + static final int MAX_LOGS_BYTES = 20 * 1024 * 1024; + private static final int MAX_RESPONSE_BYTES = 256 * 1024; + + /** + * Upload progress, 0..1. + */ + interface ProgressListener { + void onProgress(float progress); + } + + static final class Response { + final int status; + // The JSON answer, or null. + final JSONObject json; + + Response(int status, JSONObject json) { + this.status = status; + this.json = json; + } + + boolean ok() { + return GleapHttp.isSuccess(status); + } + + // The request is final (completed, skipped, cancelled or expired). + boolean gone() { + return status == 410; + } + } + + private static volatile OkHttpClient uploadClient; + + private GleapCaptureApi() { + } + + /** + * POST /claim: this device works on the request now. + */ + static Response claim(String requestId) throws IOException, JSONException { + JSONObject body = new JSONObject(); + body.put("deviceId", GleapCapture.deviceId()); + body.put("platform", GleapCapture.PLATFORM); + body.put("sdkType", GleapCapture.sdkType()); + body.put("sdkVersion", GleapCapture.sdkVersion()); + body.put("method", GleapCapture.METHOD_FRAMES); + return post(requestPath(requestId, "claim"), body.toString().getBytes(StandardCharsets.UTF_8), false); + } + + /** + * POST /event: declined, unsupported, failed or released (gives the claim back). + */ + static Response event(String requestId, String type, String reason) throws IOException, JSONException { + JSONObject body = new JSONObject(); + body.put("type", type); + if (reason != null) { + body.put("reason", reason); + } + return post(requestPath(requestId, "event"), body.toString().getBytes(StandardCharsets.UTF_8), false); + } + + /** + * POST /complete with the uploaded files. + */ + static Response complete(String requestId, JSONObject body) throws IOException { + return post(requestPath(requestId, "complete"), body.toString().getBytes(StandardCharsets.UTF_8), false); + } + + /** + * POST /logs with a gzip compressed JSON bundle. + */ + static Response postLogs(String requestId, byte[] gzippedJson) throws IOException { + return post(requestPath(requestId, "logs"), gzippedJson, true); + } + + static String requestPath(String requestId, String action) { + if (!GleapCapture.isValidRequestId(requestId)) { + throw new IllegalArgumentException("Invalid capture request id"); + } + return PATH_PREFIX + requestId + "/" + action; + } + + private static Response post(String path, byte[] body, boolean gzipped) throws IOException { + GleapSession session = currentSession(); + HttpURLConnection conn = GleapHttp.open(GleapConfig.getInstance().getApiUrl() + path, + gzipped ? GleapHttp.UPLOAD_READ_TIMEOUT_MS : GleapHttp.READ_TIMEOUT_MS); + try { + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setDoInput(true); + conn.setUseCaches(false); + conn.setRequestProperty("Api-Token", GleapConfig.getInstance().getSdkKey()); + conn.setRequestProperty("Accept", "application/json"); + conn.setRequestProperty("Content-Type", "application/json; charset=utf-8"); + if (gzipped) { + conn.setRequestProperty("Content-Encoding", "gzip"); + } + if (session != null) { + conn.setRequestProperty("Gleap-Id", session.getId()); + conn.setRequestProperty("Gleap-Hash", session.getHash()); + } + conn.setFixedLengthStreamingMode(body.length); + try (OutputStream os = conn.getOutputStream()) { + os.write(body); + } + int status = conn.getResponseCode(); + return new Response(status, readJson(conn, status)); + } finally { + conn.disconnect(); + } + } + + private static GleapSession currentSession() { + GleapSessionController controller = GleapSessionController.getInstance(); + return controller != null ? controller.getUserSession() : null; + } + + private static JSONObject readJson(HttpURLConnection conn, int status) { + InputStream stream = null; + try { + stream = GleapHttp.isSuccess(status) ? conn.getInputStream() : conn.getErrorStream(); + if (stream == null) { + return null; + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int read; + while ((read = stream.read(buffer)) != -1) { + out.write(buffer, 0, read); + if (out.size() > MAX_RESPONSE_BYTES) { + return null; + } + } + String text = new String(out.toByteArray(), StandardCharsets.UTF_8).trim(); + return text.startsWith("{") ? new JSONObject(text) : null; + } catch (Exception e) { + return null; + } finally { + if (stream != null) { + try { + stream.close(); + } catch (IOException ignore) { + } + } + } + } + + /** + * Uploads a file to /uploads/attachments (multipart field "file"), streamed from disk: the file + * is never read into memory as a whole. + * + * @param callHolder receives the call, so it can be cancelled from another thread + * @return the upload answer, {@code {"fileUrls": [...]}} + */ + static JSONObject uploadFile(File file, String fileName, String mimeType, ProgressListener progress, + AtomicReference callHolder) throws IOException { + GleapSession session = currentSession(); + RequestBody fileBody = new FileBody(file, MediaType.parse(mimeType), progress); + MultipartBody body = new MultipartBody.Builder() + .setType(MultipartBody.FORM) + .addFormDataPart("file", fileName, fileBody) + .build(); + Request.Builder request = new Request.Builder() + .url(GleapConfig.getInstance().getApiUrl() + UPLOAD_PATH) + .header("api-token", GleapConfig.getInstance().getSdkKey()) + .header("Accept", "application/json") + .post(body); + if (session != null) { + request.header("gleap-id", session.getId()); + request.header("gleap-hash", session.getHash()); + } + + OkHttpClient client = uploadClient(); + Call call = client.newCall(request.build()); + if (callHolder != null) { + callHolder.set(call); + } + try (okhttp3.Response response = call.execute()) { + if (!response.isSuccessful()) { + throw new IOException("The upload failed with HTTP " + response.code()); + } + ResponseBody responseBody = response.body(); + String text = responseBody != null ? responseBody.string() : ""; + try { + return new JSONObject(text); + } catch (JSONException e) { + throw new IOException("Unexpected upload answer", e); + } + } finally { + if (callHolder != null) { + callHolder.compareAndSet(call, null); + } + // No idle connection (and cleanup thread) stays behind. + try { + client.connectionPool().evictAll(); + } catch (Throwable ignore) { + } + } + } + + private static OkHttpClient uploadClient() { + OkHttpClient client = uploadClient; + if (client == null) { + synchronized (GleapCaptureApi.class) { + client = uploadClient; + if (client == null) { + client = new OkHttpClient.Builder() + .connectTimeout(GleapHttp.CONNECT_TIMEOUT_MS, TimeUnit.MILLISECONDS) + .writeTimeout(GleapHttp.UPLOAD_READ_TIMEOUT_MS, TimeUnit.MILLISECONDS) + .readTimeout(GleapHttp.UPLOAD_READ_TIMEOUT_MS, TimeUnit.MILLISECONDS) + .build(); + uploadClient = client; + } + } + } + return client; + } + + /** + * A file as request body, written in chunks with progress. + */ + private static final class FileBody extends RequestBody { + private static final long CHUNK = 64 * 1024; + private final File file; + private final MediaType contentType; + private final ProgressListener progress; + + FileBody(File file, MediaType contentType, ProgressListener progress) { + this.file = file; + this.contentType = contentType; + this.progress = progress; + } + + @Override + public MediaType contentType() { + return contentType; + } + + @Override + public long contentLength() { + return file.length(); + } + + @Override + public void writeTo(BufferedSink sink) throws IOException { + long total = Math.max(1, file.length()); + long written = 0; + float reported = -1; + Buffer buffer = new Buffer(); + try (Source source = Okio.source(file)) { + long read; + while ((read = source.read(buffer, CHUNK)) != -1) { + sink.write(buffer, read); + written += read; + float value = Math.min(1f, written / (float) total); + if (progress != null && value - reported >= 0.01f) { + reported = value; + try { + progress.onProgress(value); + } catch (Throwable ignore) { + } + } + } + } + } + } +} diff --git a/gleap/src/main/java/io/gleap/GleapCaptureExecutor.java b/gleap/src/main/java/io/gleap/GleapCaptureExecutor.java new file mode 100644 index 00000000..6decdc60 --- /dev/null +++ b/gleap/src/main/java/io/gleap/GleapCaptureExecutor.java @@ -0,0 +1,45 @@ +package io.gleap; + +import java.util.concurrent.Executor; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * Runs the capture work that blocks (log bundles, uploads, capture request calls) on up to two + * threads of its own: a long recording upload does not hold back the SDK's other requests or a + * log bundle. The threads only exist while there is work (they end after 30 s without). + */ +final class GleapCaptureExecutor { + private static final Executor DEFAULT = create(); + // Tests only: runs the work right away. + private static volatile Executor testExecutor; + + private GleapCaptureExecutor() { + } + + static void execute(Runnable work) { + Executor executor = testExecutor; + (executor != null ? executor : DEFAULT).execute(work); + } + + // Tests only; null restores the threads. + static void setExecutorForTesting(Executor executor) { + testExecutor = executor; + } + + private static Executor create() { + ThreadPoolExecutor executor = new ThreadPoolExecutor(2, 2, 30, TimeUnit.SECONDS, + new LinkedBlockingQueue(), new ThreadFactory() { + @Override + public Thread newThread(Runnable runnable) { + Thread thread = new Thread(runnable, "gleap-capture"); + thread.setDaemon(true); + return thread; + } + }); + executor.allowCoreThreadTimeOut(true); + return executor; + } +} diff --git a/gleap/src/main/java/io/gleap/GleapCaptureLogs.java b/gleap/src/main/java/io/gleap/GleapCaptureLogs.java new file mode 100644 index 00000000..15f19dd7 --- /dev/null +++ b/gleap/src/main/java/io/gleap/GleapCaptureLogs.java @@ -0,0 +1,469 @@ +package io.gleap; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Date; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.zip.GZIPOutputStream; + +/** + * Log bundles for capture requests: the server asks for the app's logs (kind "logs", pushed on + * the WebSocket or returned with a ping), or a screenshot / recording asks for the logs around it + * (attachLogs). The bundle carries what a ticket carries (POST /bugs/v2) with the same shapes, + * limited to the parts the request includes, and is sent gzip compressed to + * POST /v3/shared/capture-requests/{id}/logs. + *

+ * Each request id is handled once per process; the work runs on the capture threads. + */ +final class GleapCaptureLogs { + static final long FLUSH_TIMEOUT_MS = 500; + static final String KIND_LOGS = "logs"; + private static final int MAX_REMEMBERED_REQUESTS = 200; + // How long the replay frames may take to be read on the main thread. + private static final long REPLAY_SNAPSHOT_TIMEOUT_MS = 1000; + + /** + * What a bundle may contain (the request's options.include): everything but the replay by default. + */ + static final class Include { + final boolean consoleLog; + final boolean networkLogs; + final boolean customData; + final boolean metaData; + final boolean customEventLog; + final boolean replays; + + Include(boolean consoleLog, boolean networkLogs, boolean customData, boolean metaData, + boolean customEventLog, boolean replays) { + this.consoleLog = consoleLog; + this.networkLogs = networkLogs; + this.customData = customData; + this.metaData = metaData; + this.customEventLog = customEventLog; + this.replays = replays; + } + + static Include from(JSONObject include) { + if (include == null) { + include = new JSONObject(); + } + return new Include( + include.optBoolean("consoleLog", true), + include.optBoolean("networkLogs", true), + include.optBoolean("customData", true), + include.optBoolean("metaData", true), + include.optBoolean("customEventLog", true), + include.optBoolean("replays", false)); + } + } + + /** + * Where the bundle's data comes from: the collectors tickets use. + */ + interface Sources { + // Console logs are enabled in the remote config (as for tickets). + boolean consoleLogsEnabled(); + + JSONArray consoleLog(); + + JSONArray networkLogs(); + + JSONObject customData(); + + // The env data; null while it is disabled. + JSONObject metaData() throws JSONException; + + JSONArray customEventLog(); + + // Replays are enabled in the remote config. + boolean replaysEnabled(); + + // The uploaded replay frames, {"interval", "frames"}; null without frames. + JSONObject replay(); + } + + private static final Sources DEVICE = new Sources() { + @Override + public boolean consoleLogsEnabled() { + return GleapConfig.getInstance().isEnableConsoleLogs(); + } + + @Override + public JSONArray consoleLog() { + return GleapBug.getInstance().getLogs(); + } + + @Override + public JSONArray networkLogs() { + return GleapBug.getInstance().getNetworklogs(); + } + + @Override + public JSONObject customData() { + JSONObject customData = GleapBug.getInstance().getCustomData(); + try { + return (JSONObject) GleapNetworkLogSanitizer.deepCopy(customData); + } catch (Exception e) { + return customData; + } + } + + @Override + public JSONObject metaData() throws JSONException { + if (PhoneMeta.isEnvDataDisabled()) { + return null; + } + PhoneMeta phoneMeta = GleapBug.getInstance().getPhoneMeta(); + return phoneMeta != null ? phoneMeta.getJSONObj() : null; + } + + @Override + public JSONArray customEventLog() { + return GleapBug.getInstance().getCustomEventLog(); + } + + @Override + public boolean replaysEnabled() { + return GleapConfig.getInstance().isEnableReplays(); + } + + @Override + public JSONObject replay() { + return uploadReplaySnapshot(); + } + }; + + // After DEVICE: static fields are initialized in the order they are declared. + private static final GleapCaptureLogs instance = new GleapCaptureLogs(); + + // Request ids started on this device (in progress or sent). Guarded by this. + private final Set started = new LinkedHashSet<>(); + private volatile Sources sources = DEVICE; + + private GleapCaptureLogs() { + } + + static GleapCaptureLogs getInstance() { + return instance; + } + + // Tests only; null restores the device's collectors. + void setSourcesForTesting(Sources testSources) { + sources = testSources != null ? testSources : DEVICE; + } + + // Tests only. + synchronized void resetForTesting() { + started.clear(); + sources = DEVICE; + } + + /** + * The capture requests of a ping answer ({@code cr}). + */ + void onCaptureRequests(JSONArray requests) { + if (requests == null) { + return; + } + for (int i = 0; i < requests.length(); i++) { + onCaptureRequest(requests.optJSONObject(i)); + } + } + + /** + * A capture request the server pushed: {@code {id, kind, options, ticketShareToken, expiresAt}}. + * Only log requests come this way (screenshots and recordings start from the widget); each + * one is handled once. + */ + void onCaptureRequest(JSONObject request) { + try { + if (request == null || !KIND_LOGS.equals(request.optString("kind"))) { + return; + } + final String id = request.optString("id", null); + if (!GleapCapture.isValidRequestId(id) || !begin(id)) { + return; + } + JSONObject options = request.optJSONObject("options"); + final Include include = Include.from(options != null ? options.optJSONObject("include") : null); + GleapCaptureExecutor.execute(new Runnable() { + @Override + public void run() { + boolean finished = false; + try { + finished = handleLogsRequest(id, include); + } catch (Throwable error) { + GleapLog.w("Could not send the logs for a capture request", error); + } finally { + if (!finished) { + // Not delivered: the server pushes it again on the next connect. + forget(id); + } + } + } + }); + } catch (Throwable error) { + GleapErrors.report(error, "onCaptureRequest"); + } + } + + /** + * Claims the request, collects the logs and sends them. + * + * @return true when nothing is left to do for this request on this device + */ + boolean handleLogsRequest(String id, Include include) throws IOException, JSONException { + if (!GleapCapture.isRemoteLogCollectionEnabled()) { + GleapCaptureApi.event(id, "unsupported", "Remote log collection is disabled in the app."); + return true; + } + + GleapCaptureApi.Response claim = GleapCaptureApi.claim(id); + if (!claim.ok()) { + // Gone (410) or refused for good: nothing to send. Server trouble: try again later. + return !isRetryable(claim.status); + } + + JSONObject bundle = collect(include, null, null); + GleapCaptureApi.Response response = send(id, bundle); + return response.ok() || !isRetryable(response.status); + } + + private static boolean isRetryable(int status) { + return status == 408 || status == 429 || status >= 500; + } + + /** + * Sends the logs around a screenshot or recording the customer made (its attachLogs option): + * the widget claimed that request already. Does nothing while remote log collection is off. + */ + void sendForCapture(final String requestId, JSONObject include, final Date windowStart, final Date windowEnd) { + if (!GleapCapture.isRemoteLogCollectionEnabled() || !GleapCapture.isValidRequestId(requestId)) { + return; + } + final Include parts = Include.from(include); + GleapCaptureExecutor.execute(new Runnable() { + @Override + public void run() { + try { + GleapCaptureApi.Response response = send(requestId, collect(parts, windowStart, windowEnd)); + if (!response.ok()) { + GleapLog.w("The capture logs were not accepted (HTTP " + response.status + ")"); + } + } catch (Throwable error) { + GleapLog.w("Could not send the capture logs", error); + } + } + }); + } + + private JSONObject collect(Include include, Date windowStart, Date windowEnd) throws JSONException { + flushWrapperLogs(); + return buildBundle(include, sources, windowStart, windowEnd, new Date(), GleapCapture.deviceId()); + } + + /** + * The bundle: the included parts (absent = not collected), the collection time and window, + * and who sent it. + */ + static JSONObject buildBundle(Include include, Sources sources, Date windowStart, Date windowEnd, + Date capturedAt, String deviceId) throws JSONException { + JSONObject bundle = new JSONObject(); + if (include.consoleLog && sources.consoleLogsEnabled()) { + putIfPresent(bundle, "consoleLog", sources.consoleLog()); + } + if (include.networkLogs) { + putIfPresent(bundle, "networkLogs", sources.networkLogs()); + } + if (include.customData) { + putIfPresent(bundle, "customData", sources.customData()); + } + if (include.metaData) { + putIfPresent(bundle, "metaData", sources.metaData()); + } + if (include.customEventLog) { + putIfPresent(bundle, "customEventLog", sources.customEventLog()); + } + if (include.replays && sources.replaysEnabled()) { + putIfPresent(bundle, "replay", sources.replay()); + } + + String captured = DateUtil.dateToString(capturedAt); + bundle.put("capturedAt", captured); + if (windowStart != null) { + bundle.put("windowStart", DateUtil.dateToString(windowStart)); + } else { + Date oldest = oldestEntry(bundle); + bundle.put("windowStart", oldest != null && oldest.before(capturedAt) ? DateUtil.dateToString(oldest) : captured); + } + bundle.put("windowEnd", windowEnd != null ? DateUtil.dateToString(windowEnd) : captured); + bundle.put("platform", GleapCapture.PLATFORM); + bundle.put("sdkType", GleapCapture.sdkType()); + bundle.put("sdkVersion", GleapCapture.sdkVersion()); + if (deviceId != null) { + bundle.put("deviceId", deviceId); + } + return bundle; + } + + private static void putIfPresent(JSONObject bundle, String key, Object value) throws JSONException { + if (value != null) { + bundle.put(key, value); + } + } + + // The oldest dated entry of the logs and events, or null. + private static Date oldestEntry(JSONObject bundle) { + Date oldest = null; + for (String key : new String[]{"consoleLog", "networkLogs", "customEventLog"}) { + JSONArray entries = bundle.optJSONArray(key); + if (entries == null) { + continue; + } + for (int i = 0; i < entries.length(); i++) { + JSONObject entry = entries.optJSONObject(i); + Object date = entry != null ? entry.opt("date") : null; + if (!(date instanceof String)) { + continue; + } + try { + Date parsed = DateUtil.stringToDate((String) date); + if (oldest == null || parsed.before(oldest)) { + oldest = parsed; + } + } catch (Exception ignore) { + } + } + } + return oldest; + } + + /** + * Sends the bundle gzip compressed, leaving out its biggest parts while it is above the + * server's limit. + */ + static GleapCaptureApi.Response send(String id, JSONObject bundle) throws IOException { + byte[] body = gzip(bundle.toString()); + for (String key : new String[]{"replay", "consoleLog", "networkLogs", "customEventLog"}) { + if (body.length <= GleapCaptureApi.MAX_LOGS_BYTES) { + break; + } + bundle.remove(key); + body = gzip(bundle.toString()); + } + return GleapCaptureApi.postLogs(id, body); + } + + static byte[] gzip(String json) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (GZIPOutputStream gzip = new GZIPOutputStream(out)) { + gzip.write(json.getBytes(StandardCharsets.UTF_8)); + } + return out.toByteArray(); + } + + /** + * Gives the wrapper SDK (setLogFlushHandler) up to 500 ms to hand over its buffered logs. + * Never call it on the main thread: the handler runs there. + */ + static void flushWrapperLogs() { + final GleapLogFlushHandler handler = GleapCapture.getLogFlushHandler(); + if (handler == null) { + return; + } + final CountDownLatch flushed = new CountDownLatch(1); + final Runnable done = new Runnable() { + @Override + public void run() { + flushed.countDown(); + } + }; + try { + GleapMainThread.post(new Runnable() { + @Override + public void run() { + try { + handler.onFlushRequested(done); + } catch (Throwable error) { + flushed.countDown(); + GleapErrors.report(error, "logFlushHandler"); + } + } + }); + flushed.await(FLUSH_TIMEOUT_MS, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Throwable ignore) { + } + } + + /** + * Uploads the replay frames recorded so far (/uploads/sdksteps) without clearing the replay. + */ + static JSONObject uploadReplaySnapshot() { + try { + final ScreenshotReplay[][] frames = new ScreenshotReplay[1][]; + final CountDownLatch read = new CountDownLatch(1); + // The replay is written on the main thread. + GleapMainThread.post(new Runnable() { + @Override + public void run() { + try { + Replay replay = GleapBug.getInstance().getReplay(); + frames[0] = replay != null ? replay.getScreenshots() : null; + } finally { + read.countDown(); + } + } + }); + if (!read.await(REPLAY_SNAPSHOT_TIMEOUT_MS, TimeUnit.MILLISECONDS) || frames[0] == null || frames[0].length == 0) { + return null; + } + Replay replay = GleapBug.getInstance().getReplay(); + android.app.Application application = GleapInitializer.getApplication(); + if (replay == null || application == null) { + return null; + } + JSONArray uploaded = new FeedbackUploader(application).uploadReplayFrames(frames[0]); + if (uploaded.length() == 0) { + return null; + } + JSONObject result = new JSONObject(); + result.put("interval", replay.getInterval()); + result.put("frames", uploaded); + return result; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return null; + } catch (Throwable error) { + GleapLog.w("Could not upload the replay for the capture logs", error); + return null; + } + } + + private synchronized boolean begin(String id) { + if (started.contains(id)) { + return false; + } + started.add(id); + while (started.size() > MAX_REMEMBERED_REQUESTS) { + Iterator oldest = started.iterator(); + oldest.next(); + oldest.remove(); + } + return true; + } + + private synchronized void forget(String id) { + started.remove(id); + } +} diff --git a/gleap/src/main/java/io/gleap/GleapEventService.java b/gleap/src/main/java/io/gleap/GleapEventService.java index 566cbcc7..600d09b1 100644 --- a/gleap/src/main/java/io/gleap/GleapEventService.java +++ b/gleap/src/main/java/io/gleap/GleapEventService.java @@ -28,6 +28,8 @@ class GleapEventService { // One ping carries the oldest events up to these limits; the rest follows right after. static final int MAX_EVENTS_PER_PING = 100; static final int MAX_PING_BYTES = 256 * 1024; + // A ping answer bigger than this is not read for capture requests. + private static final int MAX_PING_ANSWER_BYTES = 64 * 1024; // Created with the class: getInstance() is called from several threads. private static volatile GleapEventService instance = new GleapEventService(); @@ -407,17 +409,59 @@ static PingResponse postEvents(GleapSession session, JSONArray events) throws IO body.put("opened", Gleap.getInstance().isOpened()); body.put("ws", true); body.put("sdkVersion", BuildConfig.VERSION_NAME); + // What the SDK can capture for capture requests (screenshot, recording, logs). + body.put("caps", GleapCapture.currentCapsJson()); GleapHttp.writeJson(conn, body); int status = conn.getResponseCode(); String retryAfter = conn.getHeaderField("Retry-After"); - closeBody(conn, status); + if (GleapHttp.isSuccess(status)) { + readCaptureRequests(conn); + } else { + closeBody(conn, status); + } return new PingResponse(status, retryAfter); } finally { conn.disconnect(); } } + // Without a WebSocket the log requests come with the ping answer: {"cr": [...]}. + private static void readCaptureRequests(HttpURLConnection conn) { + InputStream stream = null; + try { + stream = conn.getInputStream(); + if (stream == null) { + return; + } + java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int read; + while ((read = stream.read(buffer)) != -1) { + out.write(buffer, 0, read); + if (out.size() > MAX_PING_ANSWER_BYTES) { + return; + } + } + String text = new String(out.toByteArray(), java.nio.charset.StandardCharsets.UTF_8).trim(); + if (!text.startsWith("{")) { + return; + } + JSONArray requests = new JSONObject(text).optJSONArray("cr"); + if (requests != null && requests.length() > 0) { + GleapCaptureLogs.getInstance().onCaptureRequests(requests); + } + } catch (Exception ignore) { + } finally { + if (stream != null) { + try { + stream.close(); + } catch (Exception ignore) { + } + } + } + } + private static void closeBody(HttpURLConnection conn, int status) { try { InputStream body = status >= 400 ? conn.getErrorStream() : conn.getInputStream(); diff --git a/gleap/src/main/java/io/gleap/GleapLogFlushHandler.java b/gleap/src/main/java/io/gleap/GleapLogFlushHandler.java new file mode 100644 index 00000000..c4f94257 --- /dev/null +++ b/gleap/src/main/java/io/gleap/GleapLogFlushHandler.java @@ -0,0 +1,19 @@ +package io.gleap; + +/** + * Lets a wrapper SDK (React Native, Flutter, Capacitor) hand over the logs it buffers itself + * before the native SDK collects the logs for a capture request (see + * {@link Gleap#setLogFlushHandler(GleapLogFlushHandler)}). + */ +public interface GleapLogFlushHandler { + /** + * Called on the main thread right before the SDK collects the logs for a capture request. + * Push the buffered logs to the SDK (e.g. {@link Gleap#attachConsoleLogs} and + * {@link Gleap#attachNetworkLogs}), then call {@code done} from any thread. The SDK waits at + * most 500 ms for {@code done} and collects the logs without waiting any longer; a later call + * of {@code done} does nothing. + * + * @param done call it once the logs were handed over + */ + void onFlushRequested(Runnable done); +} diff --git a/gleap/src/main/java/io/gleap/GleapWebSocketListener.java b/gleap/src/main/java/io/gleap/GleapWebSocketListener.java index 0ee7dbd2..6356c6bf 100644 --- a/gleap/src/main/java/io/gleap/GleapWebSocketListener.java +++ b/gleap/src/main/java/io/gleap/GleapWebSocketListener.java @@ -41,11 +41,18 @@ public boolean connect() { String wsApiUrl = GleapConfig.getInstance().getWsApiUrl(); String sdkKey = GleapConfig.getInstance().getSdkKey(); - internallyConnect(wsApiUrl + "?gleapId=" + gleapSession.getId() + "&gleapHash=" + gleapSession.getHash() + "&apiKey=" + sdkKey + "&sdkVersion=" + BuildConfig.VERSION_NAME); + internallyConnect(wsApiUrl + "?gleapId=" + gleapSession.getId() + "&gleapHash=" + gleapSession.getHash() + "&apiKey=" + sdkKey + "&sdkVersion=" + BuildConfig.VERSION_NAME + + capsParameter()); return true; } + // What the SDK can capture for capture requests, e.g. "&caps=capture.screenshot,capture.logs". + static String capsParameter() { + String caps = GleapCapture.capsQueryValue(GleapCapture.currentCaps()); + return caps.isEmpty() ? "" : "&caps=" + caps; + } + private void internallyConnect(String url) { currentUrl = url; webSocket = openWebSocket(url); @@ -104,6 +111,9 @@ public void onMessage(WebSocket webSocket, String text) { if (eventName != null && eventName.equalsIgnoreCase("update")) { JSONObject data = jsonObject.getJSONObject("data"); GleapEventService.getInstance().processEventData(data); + } else if ("capture-request".equals(eventName)) { + // The server asks for the app's logs; handled whether the widget is open or not. + GleapCaptureLogs.getInstance().onCaptureRequest(jsonObject.optJSONObject("data")); } } catch (Exception exp) {} } diff --git a/gleap/src/main/java/io/gleap/Replay.java b/gleap/src/main/java/io/gleap/Replay.java index e2af3349..fb7195ae 100644 --- a/gleap/src/main/java/io/gleap/Replay.java +++ b/gleap/src/main/java/io/gleap/Replay.java @@ -29,7 +29,8 @@ public Replay(int numberOfScreenshots, int interval) { public void addScreenshot(Bitmap bitmap, String screenName) { try { if (screenshots.size() >= numberOfScreenshots) { - screenshots.getFirst().getScreenshot().recycle(); + // Dropped, not recycled: the logs of a capture request may be uploading this + // frame on another thread right now. The garbage collector frees it. screenshots.removeFirst(); } diff --git a/gleap/src/test/java/io/gleap/GleapCaptureLogsTest.java b/gleap/src/test/java/io/gleap/GleapCaptureLogsTest.java new file mode 100644 index 00000000..abe00312 --- /dev/null +++ b/gleap/src/test/java/io/gleap/GleapCaptureLogsTest.java @@ -0,0 +1,328 @@ +package io.gleap; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Date; +import java.util.zip.GZIPInputStream; + +/** + * Log bundles for capture requests: only what the request includes and the app allows, sent + * gzip compressed with the session, once per request, and never while remote log collection is + * turned off. + */ +public class GleapCaptureLogsTest { + private static final String ID = "65f0c0ffee0123456789abcd"; + private static final String PATH = "/v3/shared/capture-requests/" + ID; + + private SdkTestEnvironment sdk; + private FakeSources sources; + + @Before + public void setUp() { + sdk = new SdkTestEnvironment(); + sdk.storeSession("id-1", "hash-1"); + sdk.controller.setSessionLoaded(true); + sources = new FakeSources(); + GleapCaptureLogs.getInstance().resetForTesting(); + GleapCaptureLogs.getInstance().setSourcesForTesting(sources); + GleapCaptureExecutor.setExecutorForTesting(new java.util.concurrent.Executor() { + @Override + public void execute(Runnable command) { + command.run(); + } + }); + GleapCapture.setRemoteLogCollectionEnabled(true); + GleapCapture.setLogFlushHandler(null); + } + + @After + public void tearDown() { + GleapCaptureExecutor.setExecutorForTesting(null); + GleapCaptureLogs.getInstance().resetForTesting(); + GleapCapture.setRemoteLogCollectionEnabled(true); + sdk.tearDown(); + } + + @Test + public void aLogsRequestIsClaimedAndAnsweredWithAGzippedBundle() throws Exception { + sdk.server.respond(PATH + "/claim", 200, "{\"claimedElsewhere\":false}"); + sdk.server.respond(PATH + "/logs", 200, "{}"); + + GleapCaptureLogs.getInstance().onCaptureRequest(logsRequest(null)); + + FakeGleapServer.Request claim = sdk.server.last(PATH + "/claim"); + JSONObject claimBody = claim.bodyJson(); + assertEquals("android", claimBody.getString("platform")); + assertEquals("NATIVE", claimBody.getString("sdkType")); + assertEquals(gleap.io.gleap.BuildConfig.VERSION_NAME, claimBody.getString("sdkVersion")); + + FakeGleapServer.Request logs = sdk.server.last(PATH + "/logs"); + assertEquals("gzip", logs.headers.get("Content-Encoding")); + assertEquals(SdkTestEnvironment.SDK_KEY, logs.headers.get("Api-Token")); + assertEquals("id-1", logs.headers.get("Gleap-Id")); + assertEquals("hash-1", logs.headers.get("Gleap-Hash")); + JSONObject bundle = new JSONObject(gunzip(logs.body.toByteArray())); + assertEquals("boom", bundle.getJSONArray("consoleLog").getJSONObject(0).getString("log")); + assertEquals(1, bundle.getJSONArray("networkLogs").length()); + assertEquals("gold", bundle.getJSONObject("customData").getString("plan")); + assertEquals("Pixel", bundle.getJSONObject("metaData").getString("deviceModel")); + assertEquals("checkout", bundle.getJSONArray("customEventLog").getJSONObject(0).getString("name")); + // The replay is only sent when the request asks for it. + assertFalse(bundle.has("replay")); + assertEquals("android", bundle.getString("platform")); + // The window starts with the oldest entry. + assertEquals("2026-09-30T10:00:00.000Z", bundle.getString("windowStart")); + assertTrue(bundle.has("capturedAt")); + assertTrue(bundle.has("windowEnd")); + } + + @Test + public void onlyTheIncludedPartsAreSent() throws Exception { + sdk.server.respond(PATH + "/claim", 200, "{}"); + sdk.server.respond(PATH + "/logs", 200, "{}"); + JSONObject include = new JSONObject() + .put("consoleLog", false) + .put("networkLogs", true) + .put("customData", false) + .put("metaData", false) + .put("customEventLog", false) + .put("replays", true); + sources.replaysEnabled = true; + + GleapCaptureLogs.getInstance().onCaptureRequest(logsRequest(include)); + + JSONObject bundle = new JSONObject(gunzip(sdk.server.last(PATH + "/logs").body.toByteArray())); + assertFalse(bundle.has("consoleLog")); + assertFalse(bundle.has("customData")); + assertFalse(bundle.has("metaData")); + assertFalse(bundle.has("customEventLog")); + assertTrue(bundle.has("networkLogs")); + assertEquals(5000, bundle.getJSONObject("replay").getInt("interval")); + } + + @Test + public void theAppsSettingsApply() throws Exception { + // Console logs off in the remote config, env data disabled, replays off. + sources.consoleLogsEnabled = false; + sources.metaData = null; + sources.replaysEnabled = false; + JSONObject bundle = GleapCaptureLogs.buildBundle( + GleapCaptureLogs.Include.from(new JSONObject().put("replays", true)), sources, null, null, + new Date(), "device-1"); + + assertFalse(bundle.has("consoleLog")); + assertFalse(bundle.has("metaData")); + assertFalse(bundle.has("replay")); + assertEquals("device-1", bundle.getString("deviceId")); + } + + @Test + public void eachRequestIsSentOnce() throws Exception { + sdk.server.respond(PATH + "/claim", 200, "{}"); + sdk.server.respond(PATH + "/logs", 200, "{}"); + + GleapCaptureLogs.getInstance().onCaptureRequest(logsRequest(null)); + // The same request again, e.g. with a ping answer or after a reconnect. + GleapCaptureLogs.getInstance().onCaptureRequests(new JSONArray().put(logsRequest(null))); + + assertEquals(1, sdk.server.requestsTo(PATH + "/claim").size()); + assertEquals(1, sdk.server.requestsTo(PATH + "/logs").size()); + } + + @Test + public void aRequestThatFailedOnTheServerSideIsTriedAgainWhenItComesBack() throws Exception { + sdk.server.respond(PATH + "/claim", 200, "{}"); + sdk.server.respond(PATH + "/logs", 503, "{}"); + sdk.server.respond(PATH + "/logs", 200, "{}"); + + GleapCaptureLogs.getInstance().onCaptureRequest(logsRequest(null)); + GleapCaptureLogs.getInstance().onCaptureRequest(logsRequest(null)); + + assertEquals(2, sdk.server.requestsTo(PATH + "/logs").size()); + } + + @Test + public void withRemoteLogCollectionOffTheRequestIsAnsweredUnsupported() throws Exception { + sdk.server.respond(PATH + "/event", 200, "{}"); + GleapCapture.setRemoteLogCollectionEnabled(false); + + GleapCaptureLogs.getInstance().onCaptureRequest(logsRequest(null)); + + assertTrue(sdk.server.requestsTo(PATH + "/claim").isEmpty()); + assertTrue(sdk.server.requestsTo(PATH + "/logs").isEmpty()); + JSONObject event = sdk.server.last(PATH + "/event").bodyJson(); + assertEquals("unsupported", event.getString("type")); + + // Captures do not bring logs either. + GleapCaptureLogs.getInstance().sendForCapture(ID, null, null, new Date()); + assertTrue(sdk.server.requestsTo(PATH + "/logs").isEmpty()); + } + + @Test + public void aClosedRequestGetsNoLogs() throws Exception { + sdk.server.respond(PATH + "/claim", 410, "{\"status\":\"expired\"}"); + + GleapCaptureLogs.getInstance().onCaptureRequest(logsRequest(null)); + + assertTrue(sdk.server.requestsTo(PATH + "/logs").isEmpty()); + } + + @Test + public void onlyLogRequestsWithAValidIdAreHandled() throws Exception { + sdk.server.respond("/v3/shared/capture-requests/", 200, "{}"); + + GleapCaptureLogs.getInstance().onCaptureRequest(new JSONObject().put("id", ID).put("kind", "screenshot")); + GleapCaptureLogs.getInstance().onCaptureRequest(new JSONObject().put("id", "../../bugs").put("kind", "logs")); + GleapCaptureLogs.getInstance().onCaptureRequest(new JSONObject().put("kind", "logs")); + GleapCaptureLogs.getInstance().onCaptureRequest(null); + + assertTrue(sdk.server.requests.isEmpty()); + } + + @Test + public void theFlushHandlerRunsOnTheMainThreadBeforeTheLogsAreRead() throws Exception { + sdk.server.respond(PATH + "/claim", 200, "{}"); + sdk.server.respond(PATH + "/logs", 200, "{}"); + final boolean[] flushed = {false}; + GleapCapture.setLogFlushHandler(new GleapLogFlushHandler() { + @Override + public void onFlushRequested(Runnable done) { + flushed[0] = true; + done.run(); + } + }); + // The main thread runs the posted work right away here. + GleapMainThread.setTestExecutor(new java.util.concurrent.Executor() { + @Override + public void execute(Runnable command) { + command.run(); + } + }); + + GleapCaptureLogs.getInstance().onCaptureRequest(logsRequest(null)); + + assertTrue(flushed[0]); + assertEquals(1, sdk.server.requestsTo(PATH + "/logs").size()); + } + + @Test + public void gzipRoundTrip() throws Exception { + String json = "{\"consoleLog\":[{\"log\":\"é ✓\"}]}"; + assertEquals(json, gunzip(GleapCaptureLogs.gzip(json))); + } + + private static JSONObject logsRequest(JSONObject include) throws Exception { + JSONObject options = new JSONObject(); + if (include != null) { + options.put("include", include); + } + return new JSONObject() + .put("id", ID) + .put("kind", "logs") + .put("options", options) + .put("ticketShareToken", "share-1") + .put("expiresAt", "2026-10-01T10:00:00.000Z"); + } + + private static String gunzip(byte[] data) throws Exception { + try (InputStream in = new GZIPInputStream(new ByteArrayInputStream(data))) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int read; + while ((read = in.read(buffer)) != -1) { + out.write(buffer, 0, read); + } + return new String(out.toByteArray(), StandardCharsets.UTF_8); + } + } + + private static final class FakeSources implements GleapCaptureLogs.Sources { + boolean consoleLogsEnabled = true; + boolean replaysEnabled = false; + JSONObject metaData; + + FakeSources() { + try { + metaData = new JSONObject().put("deviceModel", "Pixel"); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + @Override + public boolean consoleLogsEnabled() { + return consoleLogsEnabled; + } + + @Override + public JSONArray consoleLog() { + try { + return new JSONArray().put(new JSONObject() + .put("date", "2026-09-30T10:00:00.000Z").put("priority", "ERROR").put("log", "boom")); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + @Override + public JSONArray networkLogs() { + try { + return new JSONArray().put(new JSONObject() + .put("date", "2026-09-30T10:01:00.000Z").put("url", "https://api.example.com").put("success", false)); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + @Override + public JSONObject customData() { + try { + return new JSONObject().put("plan", "gold"); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + @Override + public JSONObject metaData() { + return metaData; + } + + @Override + public JSONArray customEventLog() { + try { + return new JSONArray().put(new JSONObject() + .put("name", "checkout").put("date", "2026-09-30T10:02:00.000Z")); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + @Override + public boolean replaysEnabled() { + return replaysEnabled; + } + + @Override + public JSONObject replay() { + try { + return new JSONObject().put("interval", 5000).put("frames", new JSONArray()); + } catch (Exception e) { + throw new AssertionError(e); + } + } + } +} From 3ba1c0b64a75423954bdd06e044869904a05a6e0 Mon Sep 17 00:00:00 2001 From: Lukas Boehler Date: Wed, 30 Sep 2026 21:42:51 -0400 Subject: [PATCH 04/25] feat: in-app screenshots and screen recordings for capture requests The widget (Gleap messenger) can ask the app for a screenshot or a screen recording (capture-start). The widget cannot stay open while the customer uses the app, so it is closed and the request is kept by the SDK, which shows a capture bar: a small panel window of the app (no overlay permission) on the topmost window of the current activity, above dialogs and bottom sheets, moved along with every activity, rotation and window change, above system bars, cutouts and the keyboard, draggable, labelled with the widget's localized texts (English when missing) for TalkBack. Nothing is captured before the customer taps the bar; capture commands are only taken from the messenger's origin. - Screenshot: every visible window of the activity (activity, dialogs, popups; via WindowInspector on API 29+, WindowManagerGlobal below) is copied with PixelCopy on API 26+ (View.draw below, or when a copy fails), SurfaceViews are copied separately and drawn beneath their window, dialogs get their dim; FLAG_SECURE windows, password fields and views masked with Gleap.maskView are painted black; a wrapper's GetBitmapCallback (Flutter) provides the picture instead. JPEG 85, long edge at most 2560. The widget is opened again on the conversation and gets capture-image and capture-state (preview) after its ping; the logs around it are sent when the request attaches logs. - Recording (API 26+): the same window capture at the video's size, about 4 frames a second (2 to 8, keeping the main thread under 30 % of a frame interval), drawn onto a MediaCodec H.264 input surface (hardware canvas, software canvas where it misbehaves; another encoder when the default one fails) and muxed to MP4 on a background thread: long edge at most 1280, even sizes, about 1 Mbps, a key frame every 2 s, no audio, letterboxed after a rotation or resize. It pauses (and the pause is cut) while the app is in the background, stops at the request's time limit and on critical memory, then shows a preview with Send / Retake / Cancel. Send streams the file to /uploads/attachments with progress (OkHttp), completes the request, opens the widget on the conversation and reports capture-state done; the file is deleted on every path and left-overs of a dead process at the next start. - Cancel, a refused start and failures report capture-state cancelled / unsupported / failed to the widget and the matching event to the server. While a capture runs the SDK's overlay is hidden, activation methods wait and in-app messages do not pop up. Public API: setCaptureEnabled, maskView, unmaskView. Co-Authored-By: Claude Opus 5.5 --- gleap/src/main/java/io/gleap/Gleap.java | 38 + .../main/java/io/gleap/GleapCaptureBar.java | 484 ++++++++ .../io/gleap/GleapCaptureCoordinator.java | 1021 +++++++++++++++++ .../java/io/gleap/GleapCaptureGeometry.java | 123 ++ .../java/io/gleap/GleapCapturePreview.java | 278 +++++ .../java/io/gleap/GleapCaptureRequest.java | 118 ++ .../main/java/io/gleap/GleapCaptureUi.java | 119 ++ .../main/java/io/gleap/GleapDetectorUtil.java | 26 + .../main/java/io/gleap/GleapEventService.java | 3 +- .../java/io/gleap/GleapFrameRecorder.java | 768 +++++++++++++ .../main/java/io/gleap/GleapInitializer.java | 9 + .../main/java/io/gleap/GleapMainActivity.java | 28 + .../java/io/gleap/GleapOverlayManager.java | 16 + .../main/java/io/gleap/GleapWidgetBridge.java | 12 + .../java/io/gleap/GleapWindowCapture.java | 819 +++++++++++++ .../io/gleap/GleapCaptureGeometryTest.java | 104 ++ .../test/java/io/gleap/GleapCaptureTest.java | 115 ++ 17 files changed, 4080 insertions(+), 1 deletion(-) create mode 100644 gleap/src/main/java/io/gleap/GleapCaptureBar.java create mode 100644 gleap/src/main/java/io/gleap/GleapCaptureCoordinator.java create mode 100644 gleap/src/main/java/io/gleap/GleapCaptureGeometry.java create mode 100644 gleap/src/main/java/io/gleap/GleapCapturePreview.java create mode 100644 gleap/src/main/java/io/gleap/GleapCaptureRequest.java create mode 100644 gleap/src/main/java/io/gleap/GleapCaptureUi.java create mode 100644 gleap/src/main/java/io/gleap/GleapFrameRecorder.java create mode 100644 gleap/src/main/java/io/gleap/GleapWindowCapture.java create mode 100644 gleap/src/test/java/io/gleap/GleapCaptureGeometryTest.java create mode 100644 gleap/src/test/java/io/gleap/GleapCaptureTest.java diff --git a/gleap/src/main/java/io/gleap/Gleap.java b/gleap/src/main/java/io/gleap/Gleap.java index e2bec8db..a0c862c7 100644 --- a/gleap/src/main/java/io/gleap/Gleap.java +++ b/gleap/src/main/java/io/gleap/Gleap.java @@ -2,6 +2,7 @@ import android.app.Application; import android.net.Uri; +import android.view.View; import androidx.annotation.Nullable; @@ -933,6 +934,20 @@ public void setBitmapCallback(GetBitmapCallback getBitmapCallback) { GleapErrors.guard("setBitmapCallback", () -> GleapCallbacks.getInstance().setGetBitmapCallback(getBitmapCallback)); } + /** + * Enables or disables screenshots and screen recordings for capture requests: when a + * workflow, an AI agent or a teammate asks the customer in the widget to show the issue, the + * widget closes, a small bar lets the customer go to the right screen, and the SDK captures + * your app's windows once they tap Capture (or records them between Start and Stop, Android 8 + * and newer). Nothing is captured without that tap. While disabled the widget only offers to + * upload a file. Enabled by default; can be called at any time. + * + * @param enabled false to turn in-app screenshots and recordings off + */ + public void setCaptureEnabled(boolean enabled) { + GleapErrors.guard("setCaptureEnabled", () -> GleapCapture.setCaptureEnabled(enabled)); + } + /** * Enables or disables sending the app's logs for capture requests: a workflow or an AI agent * can ask for the logs of the app while it runs (no customer action), and screenshots and @@ -948,6 +963,29 @@ public void setRemoteLogCollectionEnabled(boolean enabled) { GleapErrors.guard("setRemoteLogCollectionEnabled", () -> GleapCapture.setRemoteLogCollectionEnabled(enabled)); } + /** + * Paints the view (and everything in it) black in the screenshots and screen recordings of + * capture requests, e.g. a view showing card or account details. Password fields and windows + * with {@code FLAG_SECURE} are always masked. The SDK keeps only a weak reference: a masked + * view that goes away needs no {@link #unmaskView(View)}. Views drawn by a cross-platform + * framework itself (e.g. Flutter widgets, Compose text fields) cannot be found this way: + * mask the view that hosts them. + * + * @param view the view to mask + */ + public void maskView(View view) { + GleapErrors.guard("maskView", () -> GleapCapture.maskView(view)); + } + + /** + * Shows a view masked with {@link #maskView(View)} in captures again. + * + * @param view the masked view + */ + public void unmaskView(View view) { + GleapErrors.guard("unmaskView", () -> GleapCapture.unmaskView(view)); + } + /** * For wrapper SDKs (React Native, Flutter, Capacitor): called right before the logs for a * capture request are collected, so the wrapper can hand over the logs it buffers itself diff --git a/gleap/src/main/java/io/gleap/GleapCaptureBar.java b/gleap/src/main/java/io/gleap/GleapCaptureBar.java new file mode 100644 index 00000000..9f5128ab --- /dev/null +++ b/gleap/src/main/java/io/gleap/GleapCaptureBar.java @@ -0,0 +1,484 @@ +package io.gleap; + +import static io.gleap.GleapCaptureUi.dp; + +import android.app.Activity; +import android.content.Context; +import android.graphics.Color; +import android.graphics.PixelFormat; +import android.graphics.Typeface; +import android.graphics.drawable.GradientDrawable; +import android.os.Build; +import android.os.Handler; +import android.os.IBinder; +import android.os.Looper; +import android.view.Gravity; +import android.view.MotionEvent; +import android.view.View; +import android.view.ViewConfiguration; +import android.view.ViewGroup; +import android.view.ViewTreeObserver; +import android.view.WindowInsets; +import android.view.WindowManager; +import android.widget.FrameLayout; +import android.widget.LinearLayout; +import android.widget.TextView; + +import java.lang.ref.WeakReference; + +/** + * The bar the customer uses while a capture is running: "go to the screen, then tap Capture" + * (or Start recording) with Cancel, and while recording a red dot, the time and Stop. + *

+ * It is a small panel window of the app (no overlay permission): attached to the topmost window + * of the current activity (the activity, or the dialog / bottom sheet on top of it), moved when + * that changes and attached again for every activity. It never takes the focus (the app keeps + * the keyboard) and is never part of a capture. The customer can drag it up and down. + */ +final class GleapCaptureBar { + interface Listener { + void onCaptureTapped(); + + void onStartTapped(); + + void onStopTapped(); + + void onCancelTapped(); + } + + static final int MODE_SCREENSHOT = 0; + static final int MODE_RECORD_READY = 1; + static final int MODE_RECORDING = 2; + + private static final long ANCHOR_CHECK_MS = 1000; + private static final int ATTACH_RETRIES = 30; + + private final GleapCaptureRequest request; + private final Listener listener; + private final Handler main = new Handler(Looper.getMainLooper()); + private int mode; + private boolean busy; + // How far the customer dragged the bar up, in px; kept across activities. -1: not dragged. + private int offsetPx = -1; + + // While shown. + private WeakReference activity = new WeakReference<>(null); + private WindowManager windowManager; + private WindowManager.LayoutParams params; + private FrameLayout root; + private View anchor; + private View focusSource; + private ViewTreeObserver.OnWindowFocusChangeListener focusListener; + private TextView timer; + private int attachAttempt; + + GleapCaptureBar(GleapCaptureRequest request, int mode, Listener listener) { + this.request = request; + this.mode = mode; + this.listener = listener; + } + + int getMode() { + return mode; + } + + boolean isShownOn(Activity candidate) { + return root != null && activity.get() == candidate; + } + + /** + * Shows the bar on the activity (on its topmost window), moving it from where it was. + */ + void attach(final Activity target) { + detach(); + if (target == null || target.isFinishing()) { + return; + } + activity = new WeakReference<>(target); + View decor = target.getWindow() != null ? target.getWindow().peekDecorView() : null; + if (decor == null || decor.getWindowToken() == null || decor.getWidth() <= 0) { + // The activity's window is not attached yet: try again shortly. + if (attachAttempt++ < ATTACH_RETRIES) { + main.postDelayed(new Runnable() { + @Override + public void run() { + if (activity.get() == target && root == null) { + attach(target); + } + } + }, 100); + } + return; + } + attachAttempt = 0; + + try { + View top = GleapWindowCapture.topWindow(target, decor, null); + IBinder token = top.getWindowToken(); + if (token == null) { + top = decor; + token = decor.getWindowToken(); + } + + root = buildViews(target, decor.getWidth()); + params = new WindowManager.LayoutParams( + ViewGroup.LayoutParams.WRAP_CONTENT, + ViewGroup.LayoutParams.WRAP_CONTENT, + WindowManager.LayoutParams.TYPE_APPLICATION_SUB_PANEL, + WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE + | WindowManager.LayoutParams.FLAG_NOT_TOUCH_MODAL + | WindowManager.LayoutParams.FLAG_LAYOUT_IN_SCREEN + | WindowManager.LayoutParams.FLAG_HARDWARE_ACCELERATED, + PixelFormat.TRANSLUCENT); + params.gravity = Gravity.BOTTOM | Gravity.CENTER_HORIZONTAL; + params.token = token; + params.y = offsetPx >= 0 ? offsetPx : dp(target, 12); + params.windowAnimations = android.R.style.Animation_Toast; + params.setTitle("Gleap capture"); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + // Above the system bars, the display cutout and the keyboard. + params.setFitInsetsTypes(WindowInsets.Type.systemBars() | WindowInsets.Type.ime() + | WindowInsets.Type.displayCutout()); + } + windowManager = target.getWindowManager(); + GleapWindowCapture.addOwnWindow(root); + windowManager.addView(root, params); + anchor = top; + + listenForWindowChanges(decor); + main.postDelayed(anchorCheck, ANCHOR_CHECK_MS); + announce(mode == MODE_RECORDING ? request.label("barRecording") : hintText()); + } catch (Throwable error) { + // E.g. the window went away meanwhile: the bar comes back with the next activity. + GleapLog.w("Could not show the capture bar", error); + removeWindow(); + } + } + + /** + * Takes the bar off the screen (it can be attached again). + */ + void detach() { + main.removeCallbacksAndMessages(null); + if (focusListener != null && focusSource != null) { + try { + ViewTreeObserver observer = focusSource.getViewTreeObserver(); + if (observer.isAlive()) { + observer.removeOnWindowFocusChangeListener(focusListener); + } + } catch (Throwable ignore) { + } + } + focusListener = null; + focusSource = null; + removeWindow(); + anchor = null; + } + + private void removeWindow() { + if (root != null) { + GleapWindowCapture.removeOwnWindow(root); + if (windowManager != null) { + try { + windowManager.removeViewImmediate(root); + } catch (Throwable ignore) { + // Already gone with its parent window (a dismissed dialog). + } + } + } + root = null; + timer = null; + windowManager = null; + params = null; + } + + void setMode(int newMode) { + if (mode == newMode) { + return; + } + mode = newMode; + busy = false; + Activity current = activity.get(); + if (root != null && current != null) { + attach(current); + } + } + + /** + * Disables the buttons while the SDK works (e.g. while it starts the recording). + */ + void setBusy(boolean isBusy) { + busy = isBusy; + if (root != null) { + GleapCaptureUi.setButtonsEnabled(root, !isBusy); + } + } + + void setRecordedTime(long recordedMs, int maxSec) { + if (timer == null) { + return; + } + String text = GleapCaptureGeometry.formatClock(recordedMs / 1000) + " / " + GleapCaptureGeometry.formatClock(maxSec); + timer.setText(text); + timer.setContentDescription(request.label("barRecording") + ", " + text); + } + + // Moves the bar to the window on top now (a dialog was shown or dismissed). + private final Runnable anchorCheck = new Runnable() { + @Override + public void run() { + try { + Activity current = activity.get(); + if (current == null || root == null) { + return; + } + View decor = current.getWindow().peekDecorView(); + if (decor == null) { + return; + } + View top = GleapWindowCapture.topWindow(current, decor, null); + if (top != anchor || anchor == null || anchor.getWindowToken() == null) { + attach(current); + return; + } + } catch (Throwable ignore) { + } + main.postDelayed(this, ANCHOR_CHECK_MS); + } + }; + + private void listenForWindowChanges(View decor) { + focusSource = decor; + focusListener = new ViewTreeObserver.OnWindowFocusChangeListener() { + @Override + public void onWindowFocusChanged(boolean hasFocus) { + // A dialog took or gave back the focus: look at the top window once it settled. + main.removeCallbacks(anchorCheck); + main.postDelayed(anchorCheck, 150); + } + }; + try { + decor.getViewTreeObserver().addOnWindowFocusChangeListener(focusListener); + } catch (Throwable ignore) { + } + } + + private String hintText() { + return mode == MODE_SCREENSHOT ? request.label("barScreenshotHint") : request.label("barRecordHint"); + } + + private void announce(final String text) { + final FrameLayout view = root; + if (view == null || text == null || text.isEmpty()) { + return; + } + view.postDelayed(new Runnable() { + @Override + public void run() { + try { + view.announceForAccessibility(text); + } catch (Throwable ignore) { + } + } + }, 300); + } + + // --------------------------------------------------------------------------------------------- + // Views + + private FrameLayout buildViews(Context context, int windowWidth) { + FrameLayout container = new FrameLayout(context); + // Room for the card's shadow. + int shadow = dp(context, 8); + container.setPadding(shadow, shadow, shadow, shadow); + container.setClipToPadding(false); + + LinearLayout card = new LinearLayout(context); + card.setOrientation(LinearLayout.VERTICAL); + card.setBackground(GleapCaptureUi.card(context)); + card.setElevation(dp(context, 6)); + card.setPadding(dp(context, 16), dp(context, 12), dp(context, 12), dp(context, 12)); + makeDraggable(card, context); + + FrameLayout.LayoutParams cardParams; + if (mode == MODE_RECORDING) { + cardParams = new FrameLayout.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT); + card.addView(recordingRow(context)); + } else { + int width = Math.max(dp(context, 220), Math.min(windowWidth - 2 * dp(context, 16), dp(context, 440))); + cardParams = new FrameLayout.LayoutParams(width, ViewGroup.LayoutParams.WRAP_CONTENT); + + TextView hint = GleapCaptureUi.text(context, hintText(), 14, GleapCaptureUi.SECONDARY_TEXT_COLOR, false); + hint.setMaxLines(4); + card.addView(hint, new LinearLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT)); + + LinearLayout buttons = new LinearLayout(context); + buttons.setOrientation(LinearLayout.HORIZONTAL); + buttons.setGravity(Gravity.END | Gravity.CENTER_VERTICAL); + LinearLayout.LayoutParams buttonsParams = new LinearLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT); + buttonsParams.topMargin = dp(context, 10); + + TextView cancel = GleapCaptureUi.button(context, request.label("barCancel"), Color.TRANSPARENT, GleapCaptureUi.TEXT_COLOR); + cancel.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View view) { + if (!busy) { + listener.onCancelTapped(); + } + } + }); + buttons.addView(cancel); + + final boolean screenshot = mode == MODE_SCREENSHOT; + int primaryColor = GleapCaptureUi.primaryColor(); + TextView primary = GleapCaptureUi.button(context, + screenshot ? request.label("barCapture") : request.label("barStart"), + primaryColor, GleapCaptureUi.contrastingText(primaryColor)); + primary.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View view) { + if (busy) { + return; + } + if (screenshot) { + listener.onCaptureTapped(); + } else { + listener.onStartTapped(); + } + } + }); + LinearLayout.LayoutParams primaryParams = new LinearLayout.LayoutParams( + ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT); + primaryParams.setMarginStart(dp(context, 8)); + buttons.addView(primary, primaryParams); + card.addView(buttons, buttonsParams); + } + cardParams.gravity = Gravity.CENTER_HORIZONTAL; + container.addView(card, cardParams); + if (busy) { + GleapCaptureUi.setButtonsEnabled(container, false); + } + return container; + } + + private LinearLayout recordingRow(Context context) { + LinearLayout row = new LinearLayout(context); + row.setOrientation(LinearLayout.HORIZONTAL); + row.setGravity(Gravity.CENTER_VERTICAL); + + View dot = new View(context); + GradientDrawable circle = new GradientDrawable(); + circle.setShape(GradientDrawable.OVAL); + circle.setColor(GleapCaptureUi.STOP_COLOR); + dot.setBackground(circle); + dot.setImportantForAccessibility(View.IMPORTANT_FOR_ACCESSIBILITY_NO); + LinearLayout.LayoutParams dotParams = new LinearLayout.LayoutParams(dp(context, 10), dp(context, 10)); + dotParams.setMarginEnd(dp(context, 10)); + row.addView(dot, dotParams); + dot.addOnAttachStateChangeListener(new View.OnAttachStateChangeListener() { + @Override + public void onViewAttachedToWindow(View view) { + new Blink(view).run(); + } + + @Override + public void onViewDetachedFromWindow(View view) { + view.animate().cancel(); + } + }); + + timer = GleapCaptureUi.text(context, "", 15, GleapCaptureUi.TEXT_COLOR, true); + timer.setTypeface(Typeface.MONOSPACE, Typeface.BOLD); + row.addView(timer); + setRecordedTime(0, request.maxDurationSec); + + TextView stop = GleapCaptureUi.button(context, request.label("barStop"), GleapCaptureUi.STOP_COLOR, GleapCaptureUi.TEXT_COLOR); + stop.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View view) { + if (!busy) { + listener.onStopTapped(); + } + } + }); + LinearLayout.LayoutParams stopParams = new LinearLayout.LayoutParams( + ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT); + stopParams.setMarginStart(dp(context, 16)); + row.addView(stop, stopParams); + return row; + } + + // The recording dot blinks while it is shown. + private static final class Blink implements Runnable { + private final WeakReference view; + + Blink(View view) { + this.view = new WeakReference<>(view); + } + + @Override + public void run() { + View dot = view.get(); + if (dot == null || !dot.isAttachedToWindow()) { + return; + } + dot.animate().alpha(dot.getAlpha() < 0.7f ? 1f : 0.35f).setDuration(700).withEndAction(this).start(); + } + } + + private void makeDraggable(final View card, Context context) { + final int slop = ViewConfiguration.get(context).getScaledTouchSlop(); + card.setOnTouchListener(new View.OnTouchListener() { + private float downY; + private int startOffset; + private boolean dragging; + + @Override + public boolean onTouch(View view, MotionEvent event) { + if (params == null || root == null || windowManager == null) { + return false; + } + switch (event.getActionMasked()) { + case MotionEvent.ACTION_DOWN: + downY = event.getRawY(); + startOffset = params.y; + dragging = false; + return true; + case MotionEvent.ACTION_MOVE: + float delta = downY - event.getRawY(); + if (!dragging && Math.abs(delta) > slop) { + dragging = true; + } + if (dragging) { + Activity current = activity.get(); + View decor = current != null && current.getWindow() != null ? current.getWindow().peekDecorView() : null; + int max = decor != null + ? Math.max(0, decor.getHeight() - root.getHeight() - dp(view.getContext(), 48)) + : Integer.MAX_VALUE; + params.y = Math.max(0, Math.min(max, Math.round(startOffset + delta))); + offsetPx = params.y; + try { + windowManager.updateViewLayout(root, params); + } catch (Throwable ignore) { + } + } + return true; + case MotionEvent.ACTION_UP: + if (!dragging) { + // A tap on the bar itself (not a button): nothing to do, but + // accessibility services see it as a click. + view.performClick(); + } + dragging = false; + return true; + case MotionEvent.ACTION_CANCEL: + dragging = false; + return true; + default: + return false; + } + } + }); + } +} diff --git a/gleap/src/main/java/io/gleap/GleapCaptureCoordinator.java b/gleap/src/main/java/io/gleap/GleapCaptureCoordinator.java new file mode 100644 index 00000000..b7ff06ce --- /dev/null +++ b/gleap/src/main/java/io/gleap/GleapCaptureCoordinator.java @@ -0,0 +1,1021 @@ +package io.gleap; + +import android.app.Activity; +import android.app.Application; +import android.content.ComponentCallbacks2; +import android.content.res.Configuration; +import android.graphics.Bitmap; +import android.os.Bundle; +import android.os.Handler; +import android.os.HandlerThread; +import android.os.Looper; +import android.os.SystemClock; +import android.util.Base64; +import android.view.View; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.lang.ref.WeakReference; +import java.util.ArrayList; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import okhttp3.Call; + +/** + * Runs the screenshots and recordings the widget asks for (capture-start). The widget cannot stay + * open while the customer uses the app, so it is closed and the request is kept here: + *

    + *
  1. The capture bar is shown on the app ({@link GleapCaptureBar}), on every activity the + * customer goes to, until they tap Capture / Start recording (nothing is captured without that + * tap) or Cancel.
    2. + *
  2. Screenshot: the app's windows are captured ({@link GleapWindowCapture}) and the widget is + * opened again on the conversation; its page gets capture-image and capture-state (after its ping) + * and lets the customer mark up and send the screenshot.
    3. + *
  3. Recording (API 26+): {@link GleapFrameRecorder} records until Stop, the time limit, or low + * memory; the customer previews it ({@link GleapCapturePreview}) and sends, retakes or cancels it. + * The SDK uploads it, completes the request, and opens the widget again with capture-state done.
    4. + *
+ * Only one capture runs at a time. Everything here runs on the main thread; nothing is registered + * or running while no capture runs. + */ +final class GleapCaptureCoordinator implements Application.ActivityLifecycleCallbacks, ComponentCallbacks2 { + // When the SDK looks for the app's activity itself, in case its resume was not seen. + private static final long HOST_CHECK_MS = 3000; + // Messages for the next widget page are dropped when the widget is not opened this soon. + private static final long PENDING_TTL_MS = 10 * 60 * 1000; + private static final long TIMER_TICK_MS = 500; + private static final long SCREENSHOT_TIMEOUT_MS = 10000; + private static final int JPEG_QUALITY = 85; + private static final String RECORDING_NAME = "screen-recording.mp4"; + private static final String RECORDING_TYPE = "video/mp4"; + + private static final GleapCaptureCoordinator instance = new GleapCaptureCoordinator(); + private static volatile boolean sessionActive; + + private enum State { + IDLE, + // The widget closes; the bar comes up once an activity of the app is shown. + WAITING, + BAR, + CAPTURING, + RECORDING, + STOPPING, + PREVIEW, + UPLOADING + } + + private final Handler main = new Handler(Looper.getMainLooper()); + private State state = State.IDLE; + private GleapCaptureRequest request; + private Application application; + private WeakReference host = new WeakReference<>(null); + private GleapCaptureBar bar; + private GleapCapturePreview preview; + private GleapFrameRecorder recorder; + private GleapFrameRecorder.Result recording; + private HandlerThread captureThread; + private final AtomicReference uploadCall = new AtomicReference<>(); + // Counts uploads, so the result of a cancelled one is ignored. + private int uploadGeneration; + private boolean lifecycleRegistered; + private boolean memoryRegistered; + + // Messages for the next widget page: {request id, message}. + private final List pending = new ArrayList<>(); + private long pendingUntil; + + private GleapCaptureCoordinator() { + } + + static GleapCaptureCoordinator getInstance() { + return instance; + } + + /** + * A capture is running (the widget is closed for it): the SDK's in-app messages and + * activation methods wait. + */ + static boolean isSessionActive() { + return sessionActive; + } + + // --------------------------------------------------------------------------------------------- + // The widget + + /** + * The widget answered its ping: tells it what the SDK can capture. + */ + void sendCapabilities(GleapMainActivity widget) { + try { + widget.sendMessage(GleapWidgetMessages.message("capture-capabilities", GleapCapture.currentCapabilities())); + } catch (Throwable error) { + GleapErrors.report(error, "sendCapabilities"); + } + } + + /** + * The widget answered its ping: hands it the results of a capture that ended while it was + * closed (capture-image, capture-state). + */ + void deliverPending(GleapMainActivity widget) { + try { + if (pending.isEmpty()) { + return; + } + if (SystemClock.uptimeMillis() > pendingUntil) { + pending.clear(); + return; + } + if (!widget.isTrustedWidgetPage()) { + return; + } + for (String[] message : pending) { + widget.sendMessage(message[1]); + } + pending.clear(); + } catch (Throwable error) { + GleapErrors.report(error, "deliverPending"); + } + } + + /** + * capture-start, capture-cancel, capture-done and capture-editor from the widget. Only the + * Gleap messenger page may send them. + */ + void onWidgetMessage(GleapMainActivity widget, String name, JSONObject data) { + try { + if (!widget.isTrustedWidgetPage()) { + GleapLog.w("Ignored " + name + ": it did not come from the Gleap messenger"); + return; + } + if ("capture-start".equals(name)) { + onCaptureStart(widget, data); + } else if ("capture-cancel".equals(name)) { + onCaptureEnded(data); + } else if ("capture-done".equals(name)) { + onCaptureEnded(data); + } + // capture-editor: the widget is shown full screen anyway. + } catch (Throwable error) { + GleapErrors.report(error, name); + } + } + + private void onCaptureStart(GleapMainActivity widget, JSONObject data) { + GleapCaptureRequest next = GleapCaptureRequest.fromStart(data); + if (next == null) { + return; + } + if (state != State.IDLE) { + if (request != null && request.id.equals(next.id)) { + // A repeated tap. + return; + } + // Another request replaces the one running. + releaseRunning(); + } + // Refused right away: the widget hears it now and the request's claim is given back. + if (!GleapCapture.isCaptureEnabled()) { + refuse(widget, next.id, "unsupported", "Screen capture is turned off in this app."); + return; + } + if (next.isRecording() && !GleapCapture.isRecordingSupported()) { + refuse(widget, next.id, "unsupported", "Screen recording needs Android 8 or newer."); + return; + } + Application app = GleapInitializer.getApplication(); + if (app == null) { + refuse(widget, next.id, "failed", "The SDK is not initialized."); + return; + } + + application = app; + request = next; + removePending(null); + setState(State.WAITING); + registerLifecycle(); + GleapOverlayManager.getInstance().setHiddenForCapture(true); + GleapDetectorUtil.pauseDetectorsForCapture(); + bar = new GleapCaptureBar(next, next.isRecording() ? GleapCaptureBar.MODE_RECORD_READY : GleapCaptureBar.MODE_SCREENSHOT, barListener); + sendNow(widget, stateMessage(next.id, "bar", null)); + + // The customer goes back to the app: the widget closes, the bar comes up. + widget.closeMainGleapActivity(); + main.postDelayed(hostCheck, HOST_CHECK_MS); + } + + // capture-cancel / capture-done: the widget ended the request; nothing may stay behind. + private void onCaptureEnded(JSONObject data) { + String id = data != null ? data.optString("requestId", null) : null; + if (id == null) { + return; + } + removePending(id); + if (request != null && request.id.equals(id) && state != State.IDLE) { + discardRecording(); + endSession(); + } + } + + // The bar comes up when an activity of the app resumes. Should that have been missed (the + // activity was shown already), it comes up on the current one. Without one (the app went to + // the background) it waits for the next activity; a slow device is no reason to fail. + private final Runnable hostCheck = new Runnable() { + @Override + public void run() { + if (state != State.WAITING) { + return; + } + Activity current = ActivityUtil.getCurrentActivity(); + if (current != null && !ActivityUtil.isGleapActivity(current) && !current.isFinishing()) { + onHostShown(current); + } + } + }; + + // --------------------------------------------------------------------------------------------- + // Activities + + @Override + public void onActivityResumed(Activity activity) { + try { + if (ActivityUtil.isGleapActivity(activity)) { + onWidgetOpenedDuringCapture(); + } else { + onHostShown(activity); + } + } catch (Throwable error) { + GleapErrors.report(error, "capture onActivityResumed"); + } + } + + private void onHostShown(Activity activity) { + host = new WeakReference<>(activity); + switch (state) { + case WAITING: + main.removeCallbacks(hostCheck); + setState(State.BAR); + bar.attach(activity); + break; + case BAR: + bar.attach(activity); + break; + case RECORDING: + bar.attach(activity); + recorder.setActivity(activity); + break; + case PREVIEW: + case UPLOADING: + if (preview != null && !preview.isShownOn(activity)) { + preview.show(activity); + } + break; + default: + break; + } + } + + // The app (or the customer) opened the widget while the bar was shown: that ends the capture; + // the widget learns it after its ping. A running upload goes on. + private void onWidgetOpenedDuringCapture() { + if (state == State.IDLE || state == State.WAITING || state == State.UPLOADING || request == null) { + return; + } + String id = request.id; + discardRecording(); + postEvent(id, "released", null); + queue(id, stateMessage(id, "cancelled", null)); + endSession(); + } + + @Override + public void onActivityPaused(Activity activity) { + try { + if (activity != host.get()) { + return; + } + if (bar != null && bar.isShownOn(activity)) { + bar.detach(); + } + if (state == State.RECORDING && recorder != null) { + // Nothing of the app is recorded while it is not shown. + recorder.setActivity(null); + } + if (preview != null && preview.isShownOn(activity)) { + preview.dismiss(); + } + } catch (Throwable error) { + GleapErrors.report(error, "capture onActivityPaused"); + } + } + + @Override + public void onActivityDestroyed(Activity activity) { + try { + if (bar != null && bar.isShownOn(activity)) { + bar.detach(); + } + if (preview != null && preview.isShownOn(activity)) { + preview.dismiss(); + } + if (host.get() == activity) { + host = new WeakReference<>(null); + } + } catch (Throwable ignore) { + } + } + + @Override + public void onActivityCreated(Activity activity, Bundle savedInstanceState) { + } + + @Override + public void onActivityStarted(Activity activity) { + } + + @Override + public void onActivityStopped(Activity activity) { + } + + @Override + public void onActivitySaveInstanceState(Activity activity, Bundle outState) { + } + + // --------------------------------------------------------------------------------------------- + // Memory (registered while recording) + + @Override + public void onTrimMemory(int level) { + if (level == TRIM_MEMORY_RUNNING_CRITICAL || level == TRIM_MEMORY_COMPLETE) { + stopRecordingForMemory(); + } + } + + @Override + public void onLowMemory() { + stopRecordingForMemory(); + } + + @Override + public void onConfigurationChanged(Configuration configuration) { + } + + private void stopRecordingForMemory() { + try { + if (state == State.RECORDING) { + GleapLog.w("Memory is low: the recording stops and keeps what was recorded"); + stopRecording(); + } + } catch (Throwable ignore) { + } + } + + // --------------------------------------------------------------------------------------------- + // The bar + + private final GleapCaptureBar.Listener barListener = new GleapCaptureBar.Listener() { + @Override + public void onCaptureTapped() { + guard("capture", new Runnable() { + @Override + public void run() { + takeScreenshot(); + } + }); + } + + @Override + public void onStartTapped() { + guard("startRecording", new Runnable() { + @Override + public void run() { + startRecording(); + } + }); + } + + @Override + public void onStopTapped() { + guard("stopRecording", new Runnable() { + @Override + public void run() { + stopRecording(); + } + }); + } + + @Override + public void onCancelTapped() { + guard("cancelCapture", new Runnable() { + @Override + public void run() { + cancelByCustomer(); + } + }); + } + }; + + private void guard(String context, Runnable action) { + try { + action.run(); + } catch (Throwable error) { + GleapErrors.report(error, context); + if (request != null) { + fail("Something went wrong on the device."); + } + } + } + + // --------------------------------------------------------------------------------------------- + // Screenshot + + private void takeScreenshot() { + final Activity activity = host.get(); + if (state != State.BAR || request == null || request.isRecording() || activity == null || activity.isFinishing()) { + return; + } + if (!GleapCapture.isCaptureEnabled()) { + unsupported("Screen capture is turned off in this app."); + return; + } + View decor = activity.getWindow() != null ? activity.getWindow().peekDecorView() : null; + if (decor == null || decor.getWidth() <= 0) { + return; + } + setState(State.CAPTURING); + bar.detach(); + + final String requestId = request.id; + final Date capturedAt = new Date(); + float scale = GleapCaptureGeometry.scaleForLongEdge(decor.getWidth(), decor.getHeight(), GleapCaptureGeometry.SCREENSHOT_MAX_EDGE); + main.postDelayed(screenshotTimeout, SCREENSHOT_TIMEOUT_MS); + GleapWindowCapture.capture(activity, scale, null, captureHandler(), null, new GleapWindowCapture.Callback() { + @Override + public void onFrame(GleapWindowCapture.Frame frame) { + // On the capture thread: compose and encode. + String dataUrl = null; + int width = 0; + int height = 0; + try { + if (frame != null && frame.hasContent()) { + Bitmap image = frame.render(); + width = image.getWidth(); + height = image.getHeight(); + dataUrl = jpegDataUrl(image); + image.recycle(); + } + } catch (Throwable error) { + GleapLog.w("Could not encode the screenshot", error); + } finally { + if (frame != null) { + frame.release(null); + } + } + final String result = dataUrl; + final int resultWidth = width; + final int resultHeight = height; + main.post(new Runnable() { + @Override + public void run() { + onScreenshotTaken(requestId, result, resultWidth, resultHeight, capturedAt); + } + }); + } + }); + } + + private final Runnable screenshotTimeout = new Runnable() { + @Override + public void run() { + if (state == State.CAPTURING) { + fail("The screenshot took too long."); + } + } + }; + + private void onScreenshotTaken(String requestId, String dataUrl, int width, int height, Date capturedAt) { + main.removeCallbacks(screenshotTimeout); + if (state != State.CAPTURING || request == null || !request.id.equals(requestId)) { + return; + } + if (dataUrl == null) { + fail("The screenshot could not be taken."); + return; + } + try { + JSONObject image = new JSONObject(); + image.put("requestId", requestId); + image.put("dataUrl", dataUrl); + image.put("width", width); + image.put("height", height); + image.put("method", GleapCapture.METHOD_FRAMES); + image.put("platform", GleapCapture.PLATFORM); + image.put("sdkType", GleapCapture.sdkType()); + image.put("sdkVersion", GleapCapture.sdkVersion()); + queue(requestId, GleapWidgetMessages.message("capture-image", image)); + queue(requestId, stateMessage(requestId, "preview", null)); + } catch (JSONException error) { + fail("The screenshot could not be sent to the widget."); + return; + } + if (request.attachLogs) { + GleapCaptureLogs.getInstance().sendForCapture(requestId, request.include, null, capturedAt); + } + finishAndReopen(); + } + + // JPEG, quality 85, as a data url. + private static String jpegDataUrl(Bitmap image) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + image.compress(Bitmap.CompressFormat.JPEG, JPEG_QUALITY, out); + return "data:image/jpeg;base64," + Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP); + } + + private Handler captureHandler() { + if (captureThread == null) { + captureThread = new HandlerThread("gleap-capture-screenshot"); + captureThread.start(); + } + return new Handler(captureThread.getLooper()); + } + + // --------------------------------------------------------------------------------------------- + // Recording + + private void startRecording() { + Activity activity = host.get(); + if (state != State.BAR || request == null || !request.isRecording() || activity == null || activity.isFinishing()) { + return; + } + if (!GleapCapture.isCaptureEnabled() || !GleapCapture.isRecordingSupported()) { + unsupported("Screen recording is not available."); + return; + } + File file = new File(GleapCapture.captureDir(activity.getApplicationContext()), + "recording-" + request.id + "-" + System.currentTimeMillis() + ".mp4"); + GleapFrameRecorder next = new GleapFrameRecorder(file, request.maxDurationSec, null, recorderListener); + if (!next.start(activity)) { + // The screen is not laid out yet: the customer can tap again. + return; + } + recorder = next; + setState(State.RECORDING); + recorder.setActivity(activity); + registerMemory(); + bar.setMode(GleapCaptureBar.MODE_RECORDING); + if (!bar.isShownOn(activity)) { + bar.attach(activity); + } + main.post(timerTick); + } + + private final Runnable timerTick = new Runnable() { + @Override + public void run() { + if (state != State.RECORDING || recorder == null || request == null) { + return; + } + if (bar != null) { + bar.setRecordedTime(recorder.recordedMs(), request.maxDurationSec); + } + main.postDelayed(this, TIMER_TICK_MS); + } + }; + + private void stopRecording() { + if (state != State.RECORDING || recorder == null) { + return; + } + setState(State.STOPPING); + main.removeCallbacks(timerTick); + unregisterMemory(); + if (bar != null) { + bar.detach(); + } + recorder.stop(); + } + + private final GleapFrameRecorder.Listener recorderListener = new GleapFrameRecorder.Listener() { + @Override + public void onRecordingFinished(GleapFrameRecorder.Result result) { + try { + onRecordingFinishedInternal(result); + } catch (Throwable error) { + GleapErrors.report(error, "onRecordingFinished"); + } + } + }; + + private void onRecordingFinishedInternal(GleapFrameRecorder.Result result) { + main.removeCallbacks(timerTick); + unregisterMemory(); + if ((state != State.RECORDING && state != State.STOPPING) || request == null) { + // Cancelled meanwhile. + delete(result.file); + return; + } + recorder = null; + if (!result.success && result.empty) { + // Stopped right away: ready to record again. + delete(result.file); + setState(State.BAR); + bar.setMode(GleapCaptureBar.MODE_RECORD_READY); + Activity activity = host.get(); + if (activity != null) { + bar.attach(activity); + } + return; + } + if (!result.success) { + delete(result.file); + fail(result.error != null ? result.error : "The recording failed."); + return; + } + recording = result; + setState(State.PREVIEW); + if (bar != null) { + bar.detach(); + } + preview = new GleapCapturePreview(request, result.file, result.width, result.height, previewListener); + Activity activity = host.get(); + if (activity != null && !activity.isFinishing()) { + preview.show(activity); + } + } + + private final GleapCapturePreview.Listener previewListener = new GleapCapturePreview.Listener() { + @Override + public void onSendTapped() { + guard("sendRecording", new Runnable() { + @Override + public void run() { + sendRecording(); + } + }); + } + + @Override + public void onRetakeTapped() { + guard("retakeRecording", new Runnable() { + @Override + public void run() { + retake(); + } + }); + } + + @Override + public void onCancelTapped() { + guard("cancelRecording", new Runnable() { + @Override + public void run() { + cancelByCustomer(); + } + }); + } + }; + + private void retake() { + if (state != State.PREVIEW) { + return; + } + discardRecording(); + if (preview != null) { + preview.dismiss(); + preview = null; + } + setState(State.BAR); + bar.setMode(GleapCaptureBar.MODE_RECORD_READY); + Activity activity = host.get(); + if (activity != null) { + bar.attach(activity); + } + } + + private void sendRecording() { + if (state != State.PREVIEW || recording == null || request == null) { + return; + } + setState(State.UPLOADING); + if (preview != null) { + preview.setUploading(0); + } + final int generation = ++uploadGeneration; + final GleapFrameRecorder.Result result = recording; + final String requestId = request.id; + GleapCaptureExecutor.execute(new Runnable() { + @Override + public void run() { + int status = 0; + boolean completed = false; + try { + JSONObject uploaded = GleapCaptureApi.uploadFile(result.file, RECORDING_NAME, RECORDING_TYPE, + new GleapCaptureApi.ProgressListener() { + @Override + public void onProgress(final float progress) { + main.post(new Runnable() { + @Override + public void run() { + if (generation == uploadGeneration && state == State.UPLOADING && preview != null) { + // The upload is most of the work; completing is the rest. + preview.setUploading(progress * 0.95f); + } + } + }); + } + }, uploadCall); + JSONArray urls = uploaded.optJSONArray("fileUrls"); + String url = urls != null ? urls.optString(0, null) : null; + if (url == null || url.isEmpty()) { + throw new java.io.IOException("The upload returned no file"); + } + GleapCaptureApi.Response response = GleapCaptureApi.complete(requestId, completeBody(url, result)); + status = response.status; + completed = response.ok(); + } catch (Throwable error) { + GleapLog.w("Could not send the recording", error); + } + final int finalStatus = status; + final boolean finalCompleted = completed; + main.post(new Runnable() { + @Override + public void run() { + onRecordingSent(generation, requestId, finalCompleted, finalStatus); + } + }); + } + }); + } + + private static JSONObject completeBody(String url, GleapFrameRecorder.Result result) throws JSONException { + JSONObject file = new JSONObject(); + file.put("url", url); + file.put("name", RECORDING_NAME); + file.put("type", RECORDING_TYPE); + file.put("size", result.file.length()); + file.put("width", result.width); + file.put("height", result.height); + file.put("durationMs", result.durationMs); + + JSONObject body = new JSONObject(); + body.put("files", new JSONArray().put(file)); + body.put("method", GleapCapture.METHOD_FRAMES); + body.put("platform", GleapCapture.PLATFORM); + body.put("sdkType", GleapCapture.sdkType()); + body.put("sdkVersion", GleapCapture.sdkVersion()); + body.put("deviceId", GleapCapture.deviceId()); + if (result.startedAt != null) { + body.put("recordingStartedAt", DateUtil.dateToString(result.startedAt)); + } + if (result.endedAt != null) { + body.put("recordingEndedAt", DateUtil.dateToString(result.endedAt)); + } + return body; + } + + private void onRecordingSent(int generation, String requestId, boolean completed, int status) { + if (generation != uploadGeneration || state != State.UPLOADING || request == null || !request.id.equals(requestId)) { + return; + } + if (completed || status == 409 || status == 410) { + // Sent; or the request was answered on another device, cancelled or expired meanwhile. + GleapFrameRecorder.Result result = recording; + String message = completed + ? stateMessage(requestId, "done", null) + : stateMessage(requestId, "failed", "The request is closed."); + if (completed && request.attachLogs && result != null) { + GleapCaptureLogs.getInstance().sendForCapture(requestId, request.include, result.startedAt, result.endedAt); + } + discardRecording(); + if (GleapMainActivity.deliverToOpenWidget(message)) { + endSession(); + } else { + queue(requestId, message); + finishAndReopen(); + } + return; + } + // Not sent (e.g. offline): the customer can try again or cancel. + setState(State.PREVIEW); + if (preview != null) { + preview.setError(request.label("failed")); + } + } + + // --------------------------------------------------------------------------------------------- + // Ending a capture + + private void cancelByCustomer() { + if (request == null || state == State.IDLE) { + return; + } + String id = request.id; + discardRecording(); + postEvent(id, "released", null); + queue(id, stateMessage(id, "cancelled", null)); + finishAndReopen(); + } + + private void unsupported(String reason) { + if (request == null) { + return; + } + String id = request.id; + discardRecording(); + postEvent(id, "unsupported", reason); + queue(id, stateMessage(id, "unsupported", reason)); + finishAndReopen(); + } + + private void fail(String reason) { + if (request == null) { + return; + } + String id = request.id; + discardRecording(); + postEvent(id, "failed", reason); + queue(id, stateMessage(id, "failed", reason)); + finishAndReopen(); + } + + // A new capture-start replaced this one. + private void releaseRunning() { + if (request != null) { + postEvent(request.id, "released", null); + } + discardRecording(); + endSession(); + } + + // Stops recording and uploading and deletes the recording file. + private void discardRecording() { + uploadGeneration++; + Call call = uploadCall.getAndSet(null); + if (call != null) { + call.cancel(); + } + if (recorder != null) { + recorder.cancel(); + recorder = null; + } + if (recording != null) { + delete(recording.file); + recording = null; + } + } + + private void finishAndReopen() { + String token = request != null ? request.ticketShareToken : null; + endSession(); + // Back to the conversation; its page gets the pending messages after its ping. + try { + if (token != null && !token.isEmpty()) { + Gleap.getInstance().openConversation(token); + } else { + Gleap.getInstance().open(); + } + } catch (Throwable error) { + GleapErrors.report(error, "reopenWidget"); + } + } + + private void endSession() { + main.removeCallbacks(hostCheck); + main.removeCallbacks(timerTick); + main.removeCallbacks(screenshotTimeout); + if (bar != null) { + bar.detach(); + bar = null; + } + if (preview != null) { + preview.dismiss(); + preview = null; + } + if (recorder != null) { + recorder.cancel(); + recorder = null; + } + if (captureThread != null) { + captureThread.quitSafely(); + captureThread = null; + } + unregisterMemory(); + unregisterLifecycle(); + request = null; + host = new WeakReference<>(null); + setState(State.IDLE); + try { + GleapOverlayManager.getInstance().setHiddenForCapture(false); + } catch (Throwable ignore) { + } + try { + GleapDetectorUtil.resumeDetectorsAfterCapture(); + } catch (Throwable ignore) { + } + } + + private void setState(State next) { + state = next; + sessionActive = next != State.IDLE; + } + + // --------------------------------------------------------------------------------------------- + // Helpers + + private void registerLifecycle() { + if (!lifecycleRegistered && application != null) { + application.registerActivityLifecycleCallbacks(this); + lifecycleRegistered = true; + } + } + + private void unregisterLifecycle() { + if (lifecycleRegistered && application != null) { + application.unregisterActivityLifecycleCallbacks(this); + } + lifecycleRegistered = false; + } + + private void registerMemory() { + if (!memoryRegistered && application != null) { + application.registerComponentCallbacks(this); + memoryRegistered = true; + } + } + + private void unregisterMemory() { + if (memoryRegistered && application != null) { + application.unregisterComponentCallbacks(this); + } + memoryRegistered = false; + } + + private void queue(String requestId, String message) { + if (message == null) { + return; + } + pending.add(new String[]{requestId, message}); + pendingUntil = SystemClock.uptimeMillis() + PENDING_TTL_MS; + } + + // Drops the pending messages of a request (null: all of them). + private void removePending(String requestId) { + for (Iterator it = pending.iterator(); it.hasNext(); ) { + String[] message = it.next(); + if (requestId == null || requestId.equals(message[0])) { + it.remove(); + } + } + } + + private static String stateMessage(String requestId, String state, String error) { + try { + JSONObject data = new JSONObject(); + data.put("requestId", requestId); + data.put("state", state); + if (error != null) { + data.put("error", error); + } + return GleapWidgetMessages.message("capture-state", data); + } catch (JSONException e) { + return null; + } + } + + private static void refuse(GleapMainActivity widget, String requestId, String state, String reason) { + postEvent(requestId, state, reason); + sendNow(widget, stateMessage(requestId, state, reason)); + } + + private static void sendNow(GleapMainActivity widget, String message) { + if (message != null) { + widget.sendMessage(message); + } + } + + private static void postEvent(final String requestId, final String type, final String reason) { + GleapCaptureExecutor.execute(new Runnable() { + @Override + public void run() { + try { + GleapCaptureApi.event(requestId, type, reason); + } catch (Throwable error) { + GleapLog.w("Could not report the capture event " + type, error); + } + } + }); + } + + private static void delete(File file) { + if (file != null && file.exists() && !file.delete()) { + GleapLog.w("Could not delete " + file.getName()); + } + } +} diff --git a/gleap/src/main/java/io/gleap/GleapCaptureGeometry.java b/gleap/src/main/java/io/gleap/GleapCaptureGeometry.java new file mode 100644 index 00000000..435a0589 --- /dev/null +++ b/gleap/src/main/java/io/gleap/GleapCaptureGeometry.java @@ -0,0 +1,123 @@ +package io.gleap; + +import java.util.Locale; + +/** + * The size and timing math of screenshots and recordings (no Android types, so it is unit tested). + */ +final class GleapCaptureGeometry { + // Screenshots are sent with at most this long edge. + static final int SCREENSHOT_MAX_EDGE = 2560; + // Recordings are encoded with at most this long edge. + static final int RECORDING_MAX_EDGE = 1280; + + // Recording frame rate: 4 fps, between 2 and 8 fps depending on what a frame costs. + static final long TARGET_FRAME_INTERVAL_MS = 250; + static final long MIN_FRAME_INTERVAL_MS = 125; + static final long MAX_FRAME_INTERVAL_MS = 500; + // A frame may take at most this share of the frame interval on the main thread. + static final double MAIN_THREAD_BUDGET = 0.3; + // Below this share a faster frame rate is tried. + static final double MAIN_THREAD_SPARE = 0.08; + + private GleapCaptureGeometry() { + } + + /** + * The scale that makes the long edge of {@code width x height} at most {@code maxEdge}; never + * more than 1 (no upscaling). + */ + static float scaleForLongEdge(int width, int height, int maxEdge) { + int longEdge = Math.max(width, height); + if (longEdge <= 0 || longEdge <= maxEdge) { + return 1f; + } + return maxEdge / (float) longEdge; + } + + /** + * {@code width x height} scaled so the long edge is at most {@code maxEdge}, at least 1 x 1. + */ + static int[] fitLongEdge(int width, int height, int maxEdge) { + float scale = scaleForLongEdge(width, height, maxEdge); + return new int[]{Math.max(1, Math.round(width * scale)), Math.max(1, Math.round(height * scale))}; + } + + /** + * The encoder size for a recording of {@code width x height}: the long edge at most + * {@code maxEdge}, both dimensions multiples of {@code alignment} (at least 2: H.264 needs + * even sizes) and at least one alignment step. + */ + static int[] encoderSize(int width, int height, int maxEdge, int alignment) { + int align = Math.max(2, alignment); + float scale = scaleForLongEdge(width, height, maxEdge); + // A hair above the product, so float rounding never costs a whole alignment step. + int scaledWidth = (int) Math.floor(Math.max(1, width) * (double) scale + 0.001); + int scaledHeight = (int) Math.floor(Math.max(1, height) * (double) scale + 0.001); + return new int[]{alignDown(scaledWidth, align), alignDown(scaledHeight, align)}; + } + + private static int alignDown(int value, int alignment) { + return Math.max(alignment, value - value % alignment); + } + + /** + * Where a {@code sourceWidth x sourceHeight} frame goes in a {@code targetWidth x targetHeight} + * output: scaled to fit and centered, with black bars on the other sides (letterbox). + * + * @return {left, top, width, height} + */ + static float[] letterbox(int sourceWidth, int sourceHeight, int targetWidth, int targetHeight) { + if (sourceWidth <= 0 || sourceHeight <= 0 || targetWidth <= 0 || targetHeight <= 0) { + return new float[]{0, 0, Math.max(0, targetWidth), Math.max(0, targetHeight)}; + } + float scale = Math.min(targetWidth / (float) sourceWidth, targetHeight / (float) sourceHeight); + float width = sourceWidth * scale; + float height = sourceHeight * scale; + return new float[]{(targetWidth - width) / 2f, (targetHeight - height) / 2f, width, height}; + } + + /** + * The next recording frame interval: slower while a frame costs more than + * {@link #MAIN_THREAD_BUDGET} of the interval on the main thread, faster while it costs less + * than {@link #MAIN_THREAD_SPARE} and the frames keep up, back towards 4 fps otherwise. + * + * @param mainThreadMs what the last frame cost on the main thread + * @param keptUp whether the last frame was encoded before the next one was due + */ + static long nextFrameInterval(long currentMs, long mainThreadMs, boolean keptUp) { + long interval = clampInterval(currentMs); + if (mainThreadMs > interval * MAIN_THREAD_BUDGET || !keptUp) { + return clampInterval(Math.max(interval + 50, Math.round(interval * 1.5))); + } + if (mainThreadMs < interval * MAIN_THREAD_SPARE) { + return clampInterval(interval - 25); + } + if (interval > TARGET_FRAME_INTERVAL_MS) { + return clampInterval(interval - 25); + } + return interval; + } + + private static long clampInterval(long intervalMs) { + return Math.max(MIN_FRAME_INTERVAL_MS, Math.min(MAX_FRAME_INTERVAL_MS, intervalMs)); + } + + /** + * mm:ss for the recording timer. + */ + static String formatClock(long seconds) { + long safe = Math.max(0, seconds); + return String.format(Locale.ROOT, "%02d:%02d", safe / 60, safe % 60); + } + + /** + * The recording limit: the request's value clamped to 5..180 s, 60 s when missing. + */ + static int clampDuration(int requestedSec) { + if (requestedSec <= 0) { + return GleapCapture.DEFAULT_RECORDING_SEC; + } + return Math.max(GleapCapture.MIN_RECORDING_SEC, Math.min(GleapCapture.MAX_RECORDING_SEC, requestedSec)); + } +} diff --git a/gleap/src/main/java/io/gleap/GleapCapturePreview.java b/gleap/src/main/java/io/gleap/GleapCapturePreview.java new file mode 100644 index 00000000..ee903108 --- /dev/null +++ b/gleap/src/main/java/io/gleap/GleapCapturePreview.java @@ -0,0 +1,278 @@ +package io.gleap; + +import static io.gleap.GleapCaptureUi.dp; + +import android.app.Activity; +import android.app.Dialog; +import android.graphics.Color; +import android.graphics.drawable.ColorDrawable; +import android.media.MediaPlayer; +import android.view.Gravity; +import android.view.View; +import android.view.ViewGroup; +import android.view.Window; +import android.view.WindowManager; +import android.widget.FrameLayout; +import android.widget.LinearLayout; +import android.widget.TextView; +import android.widget.VideoView; + +import java.io.File; +import java.lang.ref.WeakReference; +import java.util.Locale; + +/** + * Shows the finished recording before it is sent: the video (looping), Send, Retake and + * Cancel, and the upload progress. A dialog on the current activity; it is shown again on the + * next activity after a rotation or an activity change. + */ +final class GleapCapturePreview { + interface Listener { + void onSendTapped(); + + void onRetakeTapped(); + + void onCancelTapped(); + } + + private final GleapCaptureRequest request; + private final File video; + private final int videoWidth; + private final int videoHeight; + private final Listener listener; + + private WeakReference activity = new WeakReference<>(null); + private Dialog dialog; + private VideoView videoView; + private TextView status; + private LinearLayout buttons; + private TextView send; + private TextView retake; + // What the preview shows. + private boolean uploading; + private float progress; + private String error; + + GleapCapturePreview(GleapCaptureRequest request, File video, int videoWidth, int videoHeight, Listener listener) { + this.request = request; + this.video = video; + this.videoWidth = Math.max(1, videoWidth); + this.videoHeight = Math.max(1, videoHeight); + this.listener = listener; + } + + boolean isShownOn(Activity candidate) { + return dialog != null && activity.get() == candidate; + } + + void show(Activity target) { + dismiss(); + if (target == null || target.isFinishing()) { + return; + } + View decor = target.getWindow() != null ? target.getWindow().peekDecorView() : null; + if (decor == null || decor.getWindowToken() == null) { + return; + } + activity = new WeakReference<>(target); + try { + dialog = new Dialog(target); + dialog.requestWindowFeature(Window.FEATURE_NO_TITLE); + dialog.setCancelable(false); + dialog.setCanceledOnTouchOutside(false); + dialog.setTitle(request.label("previewTitle")); + dialog.setContentView(buildViews(target, decor.getWidth(), decor.getHeight())); + Window window = dialog.getWindow(); + if (window != null) { + window.setBackgroundDrawable(new ColorDrawable(Color.TRANSPARENT)); + window.setLayout(Math.min(decor.getWidth() - 2 * dp(target, 16), dp(target, 520)), + WindowManager.LayoutParams.WRAP_CONTENT); + window.setDimAmount(0.6f); + } + dialog.show(); + if (window != null) { + GleapWindowCapture.addOwnWindow(window.getDecorView()); + } + applyState(); + startVideo(); + } catch (Throwable error) { + GleapLog.w("Could not show the recording preview", error); + dismiss(); + } + } + + void dismiss() { + if (videoView != null) { + try { + videoView.stopPlayback(); + } catch (Throwable ignore) { + } + } + if (dialog != null) { + try { + Window window = dialog.getWindow(); + if (window != null) { + GleapWindowCapture.removeOwnWindow(window.peekDecorView()); + } + dialog.dismiss(); + } catch (Throwable ignore) { + // The activity is gone already. + } + } + dialog = null; + videoView = null; + status = null; + buttons = null; + send = null; + retake = null; + } + + void setUploading(float value) { + uploading = true; + progress = value; + error = null; + applyState(); + } + + void setError(String message) { + uploading = false; + error = message; + applyState(); + } + + private void applyState() { + if (status == null || buttons == null) { + return; + } + if (uploading) { + status.setVisibility(View.VISIBLE); + String label = request.label("uploading"); + status.setText(progress > 0 ? String.format(Locale.ROOT, "%s %d%%", label, Math.round(progress * 100)) : label); + } else if (error != null) { + status.setVisibility(View.VISIBLE); + status.setText(error); + } else { + status.setVisibility(View.GONE); + } + // Cancel stays available: it also stops an upload. + if (send != null) { + GleapCaptureUi.setButtonsEnabled(send, !uploading); + } + if (retake != null) { + GleapCaptureUi.setButtonsEnabled(retake, !uploading); + } + } + + private void startVideo() { + final VideoView view = videoView; + if (view == null) { + return; + } + view.setOnPreparedListener(new MediaPlayer.OnPreparedListener() { + @Override + public void onPrepared(MediaPlayer player) { + try { + player.setLooping(true); + player.setVolume(0f, 0f); + } catch (Throwable ignore) { + } + view.start(); + } + }); + view.setOnErrorListener(new MediaPlayer.OnErrorListener() { + @Override + public boolean onError(MediaPlayer player, int what, int extra) { + // The recording can still be sent without a preview. + return true; + } + }); + view.setVideoPath(video.getAbsolutePath()); + } + + private View buildViews(Activity context, int windowWidth, int windowHeight) { + LinearLayout card = new LinearLayout(context); + card.setOrientation(LinearLayout.VERTICAL); + card.setBackground(GleapCaptureUi.card(context)); + int padding = dp(context, 16); + card.setPadding(padding, padding, padding, dp(context, 12)); + + TextView title = GleapCaptureUi.text(context, request.label("previewTitle"), 17, GleapCaptureUi.TEXT_COLOR, true); + card.addView(title); + + // The video, fitted into the card and at most 55 % of the window high. + int cardWidth = Math.min(windowWidth - 2 * dp(context, 16), dp(context, 520)) - 2 * padding; + int maxHeight = Math.round(windowHeight * 0.55f); + float aspect = videoHeight / (float) videoWidth; + int width = Math.max(1, cardWidth); + int height = Math.round(width * aspect); + if (height > maxHeight) { + height = maxHeight; + width = Math.round(height / aspect); + } + FrameLayout videoFrame = new FrameLayout(context); + videoFrame.setBackgroundColor(Color.BLACK); + LinearLayout.LayoutParams frameParams = new LinearLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, height); + frameParams.topMargin = dp(context, 12); + videoView = new VideoView(context); + videoView.setContentDescription(request.label("previewTitle")); + FrameLayout.LayoutParams videoParams = new FrameLayout.LayoutParams(width, height); + videoParams.gravity = Gravity.CENTER; + videoFrame.addView(videoView, videoParams); + card.addView(videoFrame, frameParams); + + status = GleapCaptureUi.text(context, "", 14, GleapCaptureUi.SECONDARY_TEXT_COLOR, false); + status.setAccessibilityLiveRegion(View.ACCESSIBILITY_LIVE_REGION_POLITE); + status.setVisibility(View.GONE); + LinearLayout.LayoutParams statusParams = new LinearLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT); + statusParams.topMargin = dp(context, 10); + card.addView(status, statusParams); + + buttons = new LinearLayout(context); + buttons.setOrientation(LinearLayout.HORIZONTAL); + buttons.setGravity(Gravity.END | Gravity.CENTER_VERTICAL); + LinearLayout.LayoutParams buttonsParams = new LinearLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT); + buttonsParams.topMargin = dp(context, 12); + + TextView cancel = GleapCaptureUi.button(context, request.label("barCancel"), Color.TRANSPARENT, GleapCaptureUi.TEXT_COLOR); + cancel.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View view) { + listener.onCancelTapped(); + } + }); + buttons.addView(cancel); + + retake = GleapCaptureUi.button(context, request.label("previewRetake"), 0x33FFFFFF, GleapCaptureUi.TEXT_COLOR); + retake.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View view) { + if (!uploading) { + listener.onRetakeTapped(); + } + } + }); + LinearLayout.LayoutParams retakeParams = new LinearLayout.LayoutParams( + ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT); + retakeParams.setMarginStart(dp(context, 8)); + buttons.addView(retake, retakeParams); + + int primaryColor = GleapCaptureUi.primaryColor(); + send = GleapCaptureUi.button(context, request.label("previewSend"), primaryColor, GleapCaptureUi.contrastingText(primaryColor)); + send.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View view) { + if (!uploading) { + listener.onSendTapped(); + } + } + }); + LinearLayout.LayoutParams sendParams = new LinearLayout.LayoutParams( + ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT); + sendParams.setMarginStart(dp(context, 8)); + buttons.addView(send, sendParams); + card.addView(buttons, buttonsParams); + return card; + } +} diff --git a/gleap/src/main/java/io/gleap/GleapCaptureRequest.java b/gleap/src/main/java/io/gleap/GleapCaptureRequest.java new file mode 100644 index 00000000..ee4f82a5 --- /dev/null +++ b/gleap/src/main/java/io/gleap/GleapCaptureRequest.java @@ -0,0 +1,118 @@ +package io.gleap; + +import org.json.JSONObject; + +import java.util.HashMap; +import java.util.Map; + +/** + * A screenshot or recording the widget asked for (capture-start): the request, its options and + * the labels for the SDK's own UI (localized by the widget, English when missing). + */ +final class GleapCaptureRequest { + static final String KIND_SCREENSHOT = "screenshot"; + static final String KIND_RECORDING = "recording"; + + // Labels longer than this are cut: they are shown in the SDK's small bar. + private static final int MAX_LABEL_LENGTH = 160; + + private static final Map ENGLISH = new HashMap<>(); + + static { + ENGLISH.put("barScreenshotHint", "Go to the screen you want to show, then tap Capture."); + ENGLISH.put("barCapture", "Capture"); + ENGLISH.put("barCancel", "Cancel"); + ENGLISH.put("barRecordHint", "Go to where the issue happens, then start recording."); + ENGLISH.put("barStart", "Start recording"); + ENGLISH.put("barStop", "Stop"); + ENGLISH.put("barRecording", "Recording"); + ENGLISH.put("microphone", "Microphone"); + ENGLISH.put("previewTitle", "Send this recording?"); + ENGLISH.put("previewSend", "Send"); + ENGLISH.put("previewRetake", "Retake"); + ENGLISH.put("uploading", "Uploading…"); + ENGLISH.put("recordingInterrupted", "Recording stopped because the page changed."); + ENGLISH.put("recordAgain", "Record again"); + ENGLISH.put("permissionDenied", "Screen capture was blocked. You can upload a file instead."); + ENGLISH.put("notSupported", "Screen capture isn't available here. You can upload a file instead."); + ENGLISH.put("failed", "That didn't work. Please try again or upload a file."); + } + + final String id; + final String kind; + final String ticketShareToken; + final boolean annotate; + final boolean audio; + final boolean attachLogs; + final int maxDurationSec; + // options.include as sent (null: the defaults). + final JSONObject include; + private final JSONObject labels; + + private GleapCaptureRequest(String id, String kind, String ticketShareToken, boolean annotate, boolean audio, + boolean attachLogs, int maxDurationSec, JSONObject include, JSONObject labels) { + this.id = id; + this.kind = kind; + this.ticketShareToken = ticketShareToken; + this.annotate = annotate; + this.audio = audio; + this.attachLogs = attachLogs; + this.maxDurationSec = maxDurationSec; + this.include = include; + this.labels = labels; + } + + /** + * Reads a capture-start message's data; null when it has no valid request id or an unknown kind. + */ + static GleapCaptureRequest fromStart(JSONObject data) { + if (data == null) { + return null; + } + String id = data.optString("requestId", null); + if (!GleapCapture.isValidRequestId(id)) { + return null; + } + String kind = data.optString("kind", ""); + if (!KIND_SCREENSHOT.equals(kind) && !KIND_RECORDING.equals(kind)) { + // "any" is resolved by the widget before it starts a capture. + return null; + } + JSONObject options = data.optJSONObject("options"); + if (options == null) { + options = new JSONObject(); + } + String token = data.optString("ticketShareToken", ""); + return new GleapCaptureRequest( + id, + kind, + token != null ? token.trim() : "", + options.optBoolean("annotate", true), + options.optBoolean("audio", false), + options.optBoolean("attachLogs", true), + GleapCaptureGeometry.clampDuration(options.optInt("maxDurationSec", GleapCapture.DEFAULT_RECORDING_SEC)), + options.optJSONObject("include"), + data.optJSONObject("labels")); + } + + boolean isRecording() { + return KIND_RECORDING.equals(kind); + } + + /** + * The widget's label for {@code key}, else the English default. + */ + String label(String key) { + if (labels != null) { + Object value = labels.opt(key); + if (value instanceof String) { + String text = ((String) value).trim(); + if (!text.isEmpty()) { + return text.length() > MAX_LABEL_LENGTH ? text.substring(0, MAX_LABEL_LENGTH) : text; + } + } + } + String english = ENGLISH.get(key); + return english != null ? english : ""; + } +} diff --git a/gleap/src/main/java/io/gleap/GleapCaptureUi.java b/gleap/src/main/java/io/gleap/GleapCaptureUi.java new file mode 100644 index 00000000..d2a56a03 --- /dev/null +++ b/gleap/src/main/java/io/gleap/GleapCaptureUi.java @@ -0,0 +1,119 @@ +package io.gleap; + +import android.content.Context; +import android.content.res.ColorStateList; +import android.graphics.Color; +import android.graphics.Typeface; +import android.graphics.drawable.GradientDrawable; +import android.graphics.drawable.RippleDrawable; +import android.util.TypedValue; +import android.view.Gravity; +import android.view.View; +import android.view.ViewGroup; +import android.widget.TextView; + +import androidx.core.view.AccessibilityDelegateCompat; +import androidx.core.view.ViewCompat; +import androidx.core.view.accessibility.AccessibilityNodeInfoCompat; + +/** + * The look of the SDK's capture UI (bar and recording preview): a dark card with the widget's + * color for the main action, readable on top of any app. + */ +final class GleapCaptureUi { + static final int CARD_COLOR = 0xF2141821; + static final int STOP_COLOR = 0xFFE5484D; + static final int TEXT_COLOR = 0xFFFFFFFF; + static final int SECONDARY_TEXT_COLOR = 0xE6FFFFFF; + private static final int DEFAULT_PRIMARY = 0xFF485BFF; + + private GleapCaptureUi() { + } + + static GradientDrawable card(Context context) { + GradientDrawable background = new GradientDrawable(); + background.setColor(CARD_COLOR); + background.setCornerRadius(dp(context, 18)); + return background; + } + + static TextView text(Context context, String value, int sizeSp, int color, boolean bold) { + TextView view = new TextView(context); + view.setText(value); + view.setTextColor(color); + view.setTextSize(TypedValue.COMPLEX_UNIT_SP, sizeSp); + if (bold) { + view.setTypeface(view.getTypeface(), Typeface.BOLD); + } + return view; + } + + /** + * A button with at least a 44 x 64 dp touch target, announced as a button. + * + * @param color its background; transparent for a text button + */ + static TextView button(Context context, String label, int color, int textColor) { + TextView button = text(context, label, 14, textColor, true); + button.setGravity(Gravity.CENTER); + button.setMinHeight(dp(context, 44)); + button.setMinWidth(dp(context, 64)); + button.setPadding(dp(context, 16), dp(context, 8), dp(context, 16), dp(context, 8)); + button.setMaxLines(2); + GradientDrawable shape = new GradientDrawable(); + shape.setColor(color); + shape.setCornerRadius(dp(context, 22)); + GradientDrawable mask = new GradientDrawable(); + mask.setColor(Color.WHITE); + mask.setCornerRadius(dp(context, 22)); + button.setBackground(new RippleDrawable(ColorStateList.valueOf(0x33FFFFFF), shape, mask)); + button.setClickable(true); + button.setFocusable(true); + // Taps are ignored while another app's window covers the button. + button.setFilterTouchesWhenObscured(true); + ViewCompat.setAccessibilityDelegate(button, new AccessibilityDelegateCompat() { + @Override + public void onInitializeAccessibilityNodeInfo(View host, AccessibilityNodeInfoCompat info) { + super.onInitializeAccessibilityNodeInfo(host, info); + info.setClassName("android.widget.Button"); + } + }); + return button; + } + + /** + * Enables or disables the buttons in {@code view}. + */ + static void setButtonsEnabled(View view, boolean enabled) { + if (view.isClickable() && view instanceof TextView) { + view.setEnabled(enabled); + view.setAlpha(enabled ? 1f : 0.5f); + } + if (view instanceof ViewGroup) { + ViewGroup group = (ViewGroup) view; + for (int i = 0; i < group.getChildCount(); i++) { + setButtonsEnabled(group.getChildAt(i), enabled); + } + } + } + + // The widget's color (the dashboard's, with the color scheme applied). + static int primaryColor() { + try { + return Color.parseColor(GleapConfig.getInstance().getColor()); + } catch (Throwable ignore) { + return DEFAULT_PRIMARY; + } + } + + // Dark text on light colors, white text otherwise. + static int contrastingText(int color) { + double luminance = (0.299 * Color.red(color) + 0.587 * Color.green(color) + 0.114 * Color.blue(color)) / 255; + return luminance > 0.65 ? 0xFF111827 : 0xFFFFFFFF; + } + + static int dp(Context context, float value) { + return Math.round(TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_DIP, value, + context.getResources().getDisplayMetrics())); + } +} diff --git a/gleap/src/main/java/io/gleap/GleapDetectorUtil.java b/gleap/src/main/java/io/gleap/GleapDetectorUtil.java index 6c7bc0f9..c7378111 100644 --- a/gleap/src/main/java/io/gleap/GleapDetectorUtil.java +++ b/gleap/src/main/java/io/gleap/GleapDetectorUtil.java @@ -14,6 +14,32 @@ class GleapDetectorUtil { public static void resumeAllDetectors() { widgetOpen = false; + if (GleapCaptureCoordinator.isSessionActive()) { + // The widget closed for a capture: the activation methods wait until it is done. + return; + } + for (GleapDetector detector : GleapConfig.getInstance().getGestureDetectors()) { + detector.resume(); + } + } + + /** + * Pauses the activation methods (and replays) while a capture runs, without marking the + * widget open. + */ + static void pauseDetectorsForCapture() { + for (GleapDetector detector : GleapConfig.getInstance().getGestureDetectors()) { + detector.pause(); + } + } + + /** + * Resumes them after a capture, unless the widget is open. + */ + static void resumeDetectorsAfterCapture() { + if (widgetOpen) { + return; + } for (GleapDetector detector : GleapConfig.getInstance().getGestureDetectors()) { detector.resume(); } diff --git a/gleap/src/main/java/io/gleap/GleapEventService.java b/gleap/src/main/java/io/gleap/GleapEventService.java index 600d09b1..14bd8c96 100644 --- a/gleap/src/main/java/io/gleap/GleapEventService.java +++ b/gleap/src/main/java/io/gleap/GleapEventService.java @@ -588,7 +588,8 @@ public void run() { } if (data.has("a") && data.get("a") instanceof JSONArray) { - if (Gleap.getInstance().isOpened()) { + // Like while the widget is open, nothing pops up while the customer captures the app. + if (Gleap.getInstance().isOpened() || GleapCaptureCoordinator.isSessionActive()) { return; } diff --git a/gleap/src/main/java/io/gleap/GleapFrameRecorder.java b/gleap/src/main/java/io/gleap/GleapFrameRecorder.java new file mode 100644 index 00000000..01fb7351 --- /dev/null +++ b/gleap/src/main/java/io/gleap/GleapFrameRecorder.java @@ -0,0 +1,768 @@ +package io.gleap; + +import android.annotation.TargetApi; +import android.app.Activity; +import android.graphics.Canvas; +import android.graphics.Color; +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaCodecList; +import android.media.MediaFormat; +import android.media.MediaMuxer; +import android.os.Build; +import android.os.Bundle; +import android.os.Handler; +import android.os.HandlerThread; +import android.os.Looper; +import android.os.SystemClock; +import android.view.Surface; +import android.view.View; + +import java.io.File; +import java.lang.ref.WeakReference; +import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.Set; + +/** + * Records the app's windows into an H.264 MP4 (API 26+): the main thread captures a frame + * about 4 times a second (2 to 8, see {@link GleapCaptureGeometry#nextFrameInterval}) with + * {@link GleapWindowCapture} at the recording's scale; the frames are drawn onto the encoder's + * input surface and muxed on a background thread. No audio. + *

+ * The output size is fixed when the recording starts (long edge at most 1280, even); after a + * rotation or a window resize the frames are letterboxed into it. While no activity of the app + * is shown the recording pauses, and the pause is left out of the video. + */ +@TargetApi(Build.VERSION_CODES.O) +final class GleapFrameRecorder { + interface Listener { + /** + * The recording ended (main thread). + */ + void onRecordingFinished(Result result); + } + + static final class Result { + final File file; + final boolean success; + final long durationMs; + final int width; + final int height; + final Date startedAt; + final Date endedAt; + final String error; + // Stopped before the first frame was encoded (nothing went wrong). + final boolean empty; + + Result(File file, boolean success, long durationMs, int width, int height, Date startedAt, Date endedAt, + String error, boolean empty) { + this.file = file; + this.success = success; + this.durationMs = durationMs; + this.width = width; + this.height = height; + this.startedAt = startedAt; + this.endedAt = endedAt; + this.error = error; + this.empty = empty; + } + } + + private static final String MIME_TYPE = MediaFormat.MIMETYPE_VIDEO_AVC; + private static final int BIT_RATE = 1000000; + // The fastest adaptive rate: the bit budget per frame never exceeds ~1 Mbps. + private static final int FRAME_RATE = 8; + // The SDK asks for a key frame every 2 s of video (at any frame rate); the encoder's own + // interval is only a backstop, so the two do not double up. + private static final long KEY_FRAME_INTERVAL_US = 2000000L; + private static final int ENCODER_KEY_FRAME_INTERVAL_SEC = 10; + // A frame whose copies never came back is given up after this long. + private static final long FRAME_WATCHDOG_MS = 3000; + private static final long DRAIN_TIMEOUT_US = 10000; + private static final long END_OF_STREAM_TIMEOUT_MS = 3000; + // Encoder timestamps further off the clock than this are replaced by the SDK's own. + private static final long CLOCK_TOLERANCE_US = 5000000; + // Frames the encoder may hold before it outputs them (it never holds this many). + private static final int MAX_PENDING_TIMESTAMPS = 60; + + private final Handler main = new Handler(Looper.getMainLooper()); + private final GleapWindowCapture.BitmapPool pool = new GleapWindowCapture.BitmapPool(); + private final File file; + private final long maxDurationMs; + private final Listener listener; + private final Set excludedRoots; + + // Main thread. + private WeakReference activity = new WeakReference<>(null); + private HandlerThread thread; + private Handler encoderHandler; + private boolean running; + private boolean paused; + private long intervalMs = GleapCaptureGeometry.TARGET_FRAME_INTERVAL_MS; + private long recordedMs; + private long lastTickAt; + private int frameSequence; + private int frameInFlight = -1; + private long frameInFlightSince; + private boolean keptUp = true; + private Date startedAt; + private volatile int outputWidth; + private volatile int outputHeight; + + // Encoder thread. + private MediaCodec codec; + private Surface inputSurface; + private MediaMuxer muxer; + private int track = -1; + private boolean muxerStarted; + private boolean softwareCanvas; + private boolean ownClock; + private boolean clockChecked; + private long firstTimeUs = -1; + private long lastTimeUs = -1; + private long previousTimeUs = -1; + private long lastKeyFrameUs = Long.MIN_VALUE; + // A key frame was asked for and has not come out yet: not asked for again meanwhile. + private boolean keyFrameRequested; + private int framesWritten; + private volatile boolean released; + private final MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); + // When each submitted frame was posted (µs, monotonic): the timestamps when the encoder's are off. + private final ArrayDeque submittedAt = new ArrayDeque<>(); + // Pauses as {start, end} in µs of the monotonic clock. Guarded by itself. + private final List pauses = new ArrayList<>(); + private long pauseStartedUs = -1; + + GleapFrameRecorder(File file, int maxDurationSec, Set excludedRoots, Listener listener) { + this.file = file; + this.maxDurationMs = maxDurationSec * 1000L; + this.excludedRoots = excludedRoots; + this.listener = listener; + } + + /** + * Starts recording the activity (main thread). + * + * @return false when there is nothing to record (no laid out window) + */ + boolean start(Activity current) { + View decor = current != null && current.getWindow() != null ? current.getWindow().peekDecorView() : null; + if (decor == null || decor.getWidth() <= 0 || decor.getHeight() <= 0) { + return false; + } + final int areaWidth = decor.getWidth(); + final int areaHeight = decor.getHeight(); + int[] planned = GleapCaptureGeometry.encoderSize(areaWidth, areaHeight, GleapCaptureGeometry.RECORDING_MAX_EDGE, 2); + outputWidth = planned[0]; + outputHeight = planned[1]; + + activity = new WeakReference<>(current); + thread = new HandlerThread("gleap-capture-encoder"); + thread.start(); + encoderHandler = new Handler(thread.getLooper()); + encoderHandler.post(new Runnable() { + @Override + public void run() { + prepareEncoder(areaWidth, areaHeight); + } + }); + + startedAt = new Date(); + running = true; + lastTickAt = SystemClock.uptimeMillis(); + main.post(tick); + return true; + } + + /** + * The activity to record; null while none is shown (the recording pauses). + */ + void setActivity(Activity current) { + activity = new WeakReference<>(current); + } + + /** + * The recorded time so far, without pauses. + */ + long recordedMs() { + return recordedMs; + } + + boolean isRunning() { + return running; + } + + /** + * Stops and finishes the file; the listener gets the result (main thread). + */ + void stop() { + if (!running) { + return; + } + running = false; + main.removeCallbacks(tick); + endPause(); + final long duration = recordedMs; + final Date endedAt = new Date(); + final Date started = startedAt; + encoderHandler.post(new Runnable() { + @Override + public void run() { + final Result result = finish(duration, started, endedAt); + main.post(new Runnable() { + @Override + public void run() { + try { + listener.onRecordingFinished(result); + } catch (Throwable error) { + GleapErrors.report(error, "onRecordingFinished"); + } + } + }); + } + }); + thread.quitSafely(); + } + + /** + * Stops without a result and deletes the file (main thread). + */ + void cancel() { + running = false; + main.removeCallbacks(tick); + if (encoderHandler == null) { + return; + } + encoderHandler.post(new Runnable() { + @Override + public void run() { + releaseEncoder(); + if (muxer != null) { + try { + muxer.release(); + } catch (Throwable ignore) { + } + muxer = null; + } + deleteFile(); + pool.clear(); + } + }); + thread.quitSafely(); + } + + // --------------------------------------------------------------------------------------------- + // Main thread: capturing + + private final Runnable tick = new Runnable() { + @Override + public void run() { + try { + onTick(); + } catch (Throwable error) { + GleapLog.w("A recording frame failed", error); + if (running) { + main.postDelayed(tick, intervalMs); + } + } + } + }; + + private void onTick() { + if (!running) { + return; + } + long now = SystemClock.uptimeMillis(); + Activity current = activity.get(); + if (current == null || current.isFinishing()) { + // The app is in the background (or between two activities): pause. + if (!paused) { + paused = true; + recordedMs += now - lastTickAt; + startPause(); + } + lastTickAt = now; + main.postDelayed(tick, intervalMs); + return; + } + if (paused) { + paused = false; + endPause(); + } else { + recordedMs += now - lastTickAt; + } + lastTickAt = now; + + if (recordedMs >= maxDurationMs) { + stop(); + return; + } + + if (frameInFlight >= 0) { + if (now - frameInFlightSince < FRAME_WATCHDOG_MS) { + // The last frame is still being copied or encoded: skip this one. + keptUp = false; + main.postDelayed(tick, intervalMs); + return; + } + frameInFlight = -1; + } + + final int sequence = ++frameSequence; + frameInFlight = sequence; + frameInFlightSince = now; + GleapWindowCapture.capture(current, captureScale(current), excludedRoots, encoderHandler, pool, + new GleapWindowCapture.Callback() { + @Override + public void onFrame(GleapWindowCapture.Frame frame) { + onFrameCaptured(sequence, frame); + } + }); + long mainThreadMs = SystemClock.uptimeMillis() - now; + intervalMs = GleapCaptureGeometry.nextFrameInterval(intervalMs, mainThreadMs, keptUp); + keptUp = true; + main.postDelayed(tick, intervalMs); + } + + // Frames are captured at the size they are encoded at. + private float captureScale(Activity current) { + View decor = current.getWindow() != null ? current.getWindow().peekDecorView() : null; + if (decor == null || decor.getWidth() <= 0 || decor.getHeight() <= 0) { + return 1f; + } + float scale = Math.min(outputWidth / (float) decor.getWidth(), outputHeight / (float) decor.getHeight()); + return Math.max(0.05f, Math.min(1f, scale)); + } + + private void startPause() { + synchronized (pauses) { + pauseStartedUs = nowUs(); + } + } + + private void endPause() { + synchronized (pauses) { + if (pauseStartedUs >= 0) { + pauses.add(new long[]{pauseStartedUs, nowUs()}); + pauseStartedUs = -1; + } + } + } + + private static long nowUs() { + return System.nanoTime() / 1000; + } + + // --------------------------------------------------------------------------------------------- + // Encoder thread + + private void onFrameCaptured(final int sequence, GleapWindowCapture.Frame frame) { + try { + if (frame != null && !released && codec != null) { + encode(frame); + } + } catch (Throwable error) { + GleapLog.w("Could not encode a recording frame", error); + } finally { + if (frame != null) { + frame.release(released ? null : pool); + } + main.post(new Runnable() { + @Override + public void run() { + if (frameInFlight == sequence) { + frameInFlight = -1; + } + } + }); + } + } + + private void prepareEncoder(int areaWidth, int areaHeight) { + try { + codec = createEncoder(areaWidth, areaHeight); + muxer = new MediaMuxer(file.getAbsolutePath(), MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); + softwareCanvas = prefersSoftwareCanvas(); + } catch (Throwable error) { + GleapLog.w("Could not start the recording encoder", error); + releaseEncoder(); + main.post(new Runnable() { + @Override + public void run() { + failEarly("The video encoder could not be started."); + } + }); + } + } + + // The encoder did not start: the recording ends right away. + private void failEarly(final String error) { + if (!running) { + return; + } + running = false; + main.removeCallbacks(tick); + final Date started = startedAt; + encoderHandler.post(new Runnable() { + @Override + public void run() { + released = true; + if (muxer != null) { + try { + muxer.release(); + } catch (Throwable ignore) { + } + muxer = null; + } + deleteFile(); + pool.clear(); + } + }); + thread.quitSafely(); + try { + listener.onRecordingFinished(new Result(file, false, 0, outputWidth, outputHeight, started, new Date(), error, false)); + } catch (Throwable reportError) { + GleapErrors.report(reportError, "onRecordingFinished"); + } + } + + /** + * The device's default H.264 encoder, else any other (software ones first) that accepts + * the recording's size. + */ + private MediaCodec createEncoder(int areaWidth, int areaHeight) throws Exception { + String triedName = null; + try { + MediaCodec encoder = MediaCodec.createEncoderByType(MIME_TYPE); + triedName = encoder.getName(); + if (startEncoder(encoder, areaWidth, areaHeight)) { + return encoder; + } + } catch (Throwable error) { + GleapLog.w("The default video encoder is not available", error); + } + + List candidates = new ArrayList<>(); + for (MediaCodecInfo info : new MediaCodecList(MediaCodecList.REGULAR_CODECS).getCodecInfos()) { + if (info.isEncoder() && supports(info, MIME_TYPE) && !info.getName().equals(triedName)) { + candidates.add(info); + } + } + // Software encoders first: the hardware one (the default) already failed. + List ordered = new ArrayList<>(); + for (MediaCodecInfo info : candidates) { + if (isSoftware(info)) { + ordered.add(info); + } + } + for (MediaCodecInfo info : candidates) { + if (!isSoftware(info)) { + ordered.add(info); + } + } + for (MediaCodecInfo info : ordered) { + try { + MediaCodec encoder = MediaCodec.createByCodecName(info.getName()); + if (startEncoder(encoder, areaWidth, areaHeight)) { + return encoder; + } + } catch (Throwable error) { + GleapLog.w("The video encoder " + info.getName() + " is not available", error); + } + } + throw new IllegalStateException("No H.264 encoder could be started"); + } + + private boolean startEncoder(MediaCodec encoder, int areaWidth, int areaHeight) { + try { + int[] size = encoderSize(encoder, areaWidth, areaHeight); + MediaFormat format = MediaFormat.createVideoFormat(MIME_TYPE, size[0], size[1]); + format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); + format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE); + format.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE); + format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, ENCODER_KEY_FRAME_INTERVAL_SEC); + encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + Surface surface = encoder.createInputSurface(); + encoder.start(); + inputSurface = surface; + outputWidth = size[0]; + outputHeight = size[1]; + return true; + } catch (Throwable error) { + GleapLog.w("Could not configure the video encoder " + safeName(encoder), error); + try { + encoder.release(); + } catch (Throwable ignore) { + } + return false; + } + } + + // The size the encoder takes: its alignment, and a smaller size if it cannot do the planned one. + private static int[] encoderSize(MediaCodec encoder, int areaWidth, int areaHeight) { + int alignment = 2; + MediaCodecInfo.VideoCapabilities video = null; + try { + video = encoder.getCodecInfo().getCapabilitiesForType(MIME_TYPE).getVideoCapabilities(); + alignment = Math.max(alignment, Math.max(video.getWidthAlignment(), video.getHeightAlignment())); + } catch (Throwable ignore) { + } + int[] size = GleapCaptureGeometry.encoderSize(areaWidth, areaHeight, GleapCaptureGeometry.RECORDING_MAX_EDGE, alignment); + if (video == null) { + return size; + } + int[] limits = {GleapCaptureGeometry.RECORDING_MAX_EDGE, 960, 720, 640}; + for (int limit : limits) { + for (int align : new int[]{alignment, 16}) { + int[] candidate = GleapCaptureGeometry.encoderSize(areaWidth, areaHeight, limit, align); + try { + if (video.isSizeSupported(candidate[0], candidate[1])) { + return candidate; + } + } catch (Throwable ignore) { + } + } + } + return size; + } + + private static boolean supports(MediaCodecInfo info, String mimeType) { + for (String type : info.getSupportedTypes()) { + if (type.equalsIgnoreCase(mimeType)) { + return true; + } + } + return false; + } + + private static boolean isSoftware(MediaCodecInfo info) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + return info.isSoftwareOnly(); + } + String name = info.getName().toLowerCase(Locale.ROOT); + return name.startsWith("omx.google.") || name.startsWith("c2.android."); + } + + private static String safeName(MediaCodec encoder) { + try { + return encoder.getName(); + } catch (Throwable error) { + return ""; + } + } + + // The hardware canvas draws garbage on the encoder surface of some devices. + private static boolean prefersSoftwareCanvas() { + String manufacturer = Build.MANUFACTURER != null ? Build.MANUFACTURER.toLowerCase(Locale.ROOT) : ""; + if (manufacturer.contains("xiaomi") || manufacturer.contains("motorola")) { + return true; + } + return Build.VERSION.SDK_INT >= Build.VERSION_CODES.S && "T606".equalsIgnoreCase(Build.SOC_MODEL); + } + + private void encode(GleapWindowCapture.Frame frame) { + if (inputSurface == null || !frame.hasContent()) { + return; + } + requestKeyFrameIfDue(); + Canvas canvas = lockCanvas(); + if (canvas == null) { + return; + } + try { + canvas.drawColor(Color.BLACK); + float[] box = GleapCaptureGeometry.letterbox(frame.width, frame.height, outputWidth, outputHeight); + frame.draw(canvas, box[0], box[1], box[2] / frame.width); + } finally { + long postedAt = nowUs(); + inputSurface.unlockCanvasAndPost(canvas); + submittedAt.addLast(postedAt); + while (submittedAt.size() > MAX_PENDING_TIMESTAMPS) { + submittedAt.pollFirst(); + } + } + drain(false); + } + + private Canvas lockCanvas() { + if (!softwareCanvas) { + try { + return inputSurface.lockHardwareCanvas(); + } catch (Throwable error) { + GleapLog.w("The hardware canvas is not available for the recording", error); + softwareCanvas = true; + } + } + try { + return inputSurface.lockCanvas(null); + } catch (Throwable error) { + GleapLog.w("Could not draw a recording frame", error); + return null; + } + } + + // A key frame at least every 2 s of video, also at 2 fps. + private void requestKeyFrameIfDue() { + if (keyFrameRequested || lastKeyFrameUs == Long.MIN_VALUE || lastTimeUs < 0) { + return; + } + // Asked for one frame ahead: the frame drawn now comes out about one interval later. + long frameIntervalUs = previousTimeUs >= 0 ? Math.max(0, lastTimeUs - previousTimeUs) : 0; + if (lastTimeUs + frameIntervalUs - lastKeyFrameUs < KEY_FRAME_INTERVAL_US) { + return; + } + try { + Bundle parameters = new Bundle(); + parameters.putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0); + codec.setParameters(parameters); + keyFrameRequested = true; + } catch (Throwable ignore) { + } + } + + private void drain(boolean endOfStream) { + long deadline = SystemClock.uptimeMillis() + END_OF_STREAM_TIMEOUT_MS; + while (true) { + int index = codec.dequeueOutputBuffer(bufferInfo, endOfStream ? DRAIN_TIMEOUT_US : 0); + if (index == MediaCodec.INFO_TRY_AGAIN_LATER) { + if (!endOfStream || SystemClock.uptimeMillis() > deadline) { + return; + } + continue; + } + if (index == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + if (!muxerStarted) { + track = muxer.addTrack(codec.getOutputFormat()); + muxer.start(); + muxerStarted = true; + } + continue; + } + if (index < 0) { + continue; + } + ByteBuffer data = codec.getOutputBuffer(index); + boolean codecConfig = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0; + if (data != null && bufferInfo.size > 0 && !codecConfig && muxerStarted) { + long timeUs = videoTimeUs(bufferInfo.presentationTimeUs); + bufferInfo.presentationTimeUs = timeUs; + data.position(bufferInfo.offset); + data.limit(bufferInfo.offset + bufferInfo.size); + muxer.writeSampleData(track, data, bufferInfo); + framesWritten++; + if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0) { + lastKeyFrameUs = timeUs; + keyFrameRequested = false; + } + } + codec.releaseOutputBuffer(index, false); + if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + return; + } + } + } + + /** + * The video time of an encoded frame: its capture time since the first frame, minus the + * pauses before it; strictly increasing. + */ + private long videoTimeUs(long encoderTimeUs) { + Long submitted = submittedAt.pollFirst(); + if (!clockChecked) { + clockChecked = true; + // The input surface stamps frames with the monotonic clock; if this device does not, + // the SDK's own submit times are used. + ownClock = submitted == null || Math.abs(encoderTimeUs - submitted) > CLOCK_TOLERANCE_US; + } + long captureUs = ownClock && submitted != null ? submitted : encoderTimeUs; + if (firstTimeUs < 0) { + firstTimeUs = captureUs; + } + long pausedUs = 0; + synchronized (pauses) { + for (long[] pause : pauses) { + if (pause[1] <= captureUs && pause[0] >= firstTimeUs) { + pausedUs += pause[1] - pause[0]; + } + } + } + long timeUs = Math.max(0, captureUs - firstTimeUs - pausedUs); + if (timeUs <= lastTimeUs) { + timeUs = lastTimeUs + 1; + } + previousTimeUs = lastTimeUs; + lastTimeUs = timeUs; + return timeUs; + } + + private Result finish(long durationMs, Date started, Date endedAt) { + boolean success = false; + String error = null; + try { + if (codec != null) { + codec.signalEndOfInputStream(); + drain(true); + } + } catch (Throwable drainError) { + GleapLog.w("Could not finish the recording", drainError); + } + releaseEncoder(); + try { + if (muxer != null && muxerStarted && framesWritten > 0) { + muxer.stop(); + success = true; + } else { + error = "No frame was recorded."; + } + } catch (Throwable stopError) { + error = "The recording could not be finished."; + GleapLog.w(error, stopError); + } + if (muxer != null) { + try { + muxer.release(); + } catch (Throwable ignore) { + } + muxer = null; + } + pool.clear(); + if (!success) { + deleteFile(); + } + return new Result(file, success, durationMs, outputWidth, outputHeight, started, endedAt, error, + !success && framesWritten == 0); + } + + private void releaseEncoder() { + released = true; + if (codec != null) { + try { + codec.stop(); + } catch (Throwable ignore) { + } + try { + codec.release(); + } catch (Throwable ignore) { + } + codec = null; + } + if (inputSurface != null) { + try { + inputSurface.release(); + } catch (Throwable ignore) { + } + inputSurface = null; + } + } + + private void deleteFile() { + if (file.exists() && !file.delete()) { + GleapLog.w("Could not delete the recording"); + } + } +} diff --git a/gleap/src/main/java/io/gleap/GleapInitializer.java b/gleap/src/main/java/io/gleap/GleapInitializer.java index 30f663f1..88855143 100644 --- a/gleap/src/main/java/io/gleap/GleapInitializer.java +++ b/gleap/src/main/java/io/gleap/GleapInitializer.java @@ -62,6 +62,15 @@ static void initialize(String sdkKey, Application app) { */ static void startLoading(OnHttpResponseListener configListener) { try { + // A recording left behind by a process that ended while it was recorded or sent. + final Application app = application; + GleapExecutor.SERIAL.execute(new Runnable() { + @Override + public void run() { + GleapCapture.deleteCaptureFiles(app); + } + }); + new ConfigLoader(configListener).executeOnExecutor(GleapExecutor.SERIAL, GleapBug.getInstance()); GleapBaseSessionService sessionLoader = new GleapBaseSessionService(); diff --git a/gleap/src/main/java/io/gleap/GleapMainActivity.java b/gleap/src/main/java/io/gleap/GleapMainActivity.java index 61393858..eeb11a1c 100644 --- a/gleap/src/main/java/io/gleap/GleapMainActivity.java +++ b/gleap/src/main/java/io/gleap/GleapMainActivity.java @@ -648,4 +648,32 @@ public void sendMessage(String message) { webView.evaluateJavascript("sendMessage(" + message + ");", null); } } + + /** + * Whether the WebView shows the Gleap messenger (the widget url's scheme, host and port). + * Capture commands are only taken from it. Main thread only. + */ + boolean isTrustedWidgetPage() { + try { + return webView != null && !isFinishing() + && GleapCapture.sameOrigin(webView.getUrl(), GleapConfig.getInstance().getiFrameUrl()); + } catch (Throwable error) { + return false; + } + } + + /** + * Sends a message to the open widget, if one is open, answered its ping and shows the + * messenger. Main thread only. + * + * @return whether it was sent + */ + static boolean deliverToOpenWidget(String message) { + GleapMainActivity activity = openInstance != null ? openInstance.get() : null; + if (activity == null || !activity.widgetPinged || !activity.isTrustedWidgetPage()) { + return false; + } + activity.sendMessage(message); + return true; + } } diff --git a/gleap/src/main/java/io/gleap/GleapOverlayManager.java b/gleap/src/main/java/io/gleap/GleapOverlayManager.java index f84e2cc5..54bd8316 100644 --- a/gleap/src/main/java/io/gleap/GleapOverlayManager.java +++ b/gleap/src/main/java/io/gleap/GleapOverlayManager.java @@ -44,6 +44,9 @@ class GleapOverlayManager { private GleapModal modal; private JSONObject modalData; private int originalVisibility = 0; + // While a capture runs the overlay (button, notifications, banner, modal) is hidden: it is + // not part of the app the customer shows. + private boolean hiddenForCapture = false; private GleapOverlayManager() { } @@ -91,6 +94,16 @@ public void setInvisible() { } } + /** + * Hides the whole overlay while a capture runs (main thread). + */ + void setHiddenForCapture(boolean hidden) { + hiddenForCapture = hidden; + if (layout != null) { + layout.setVisibility(hidden ? View.INVISIBLE : View.VISIBLE); + } + } + public void setVisible() { if (button.container != null && !GleapConfig.getInstance().isHideFeedbackButton()) { button.container.setVisibility(View.VISIBLE); @@ -311,6 +324,9 @@ public void addLayoutToActivity(Activity activity) { LayoutInflater inflater = activity.getLayoutInflater(); this.layout = (ConstraintLayout) inflater.inflate(R.layout.activity_gleap_fab, null); this.layout.setLayoutParams(new RelativeLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT)); + if (hiddenForCapture) { + this.layout.setVisibility(View.INVISIBLE); + } addLocalLayoutToActivity(activity); } diff --git a/gleap/src/main/java/io/gleap/GleapWidgetBridge.java b/gleap/src/main/java/io/gleap/GleapWidgetBridge.java index 09b79639..4d097847 100644 --- a/gleap/src/main/java/io/gleap/GleapWidgetBridge.java +++ b/gleap/src/main/java/io/gleap/GleapWidgetBridge.java @@ -43,9 +43,12 @@ public void run() { case "ping": sendConfigUpdate(activity); sendSessionUpdate(activity); + GleapCaptureCoordinator.getInstance().sendCapabilities(activity); sendPrefillData(activity); sendScreenshotUpdate(activity); sendPendingActions(activity); + // The result of a capture that ended while the widget was closed. + GleapCaptureCoordinator.getInstance().deliverPending(activity); GleapMainThread.postDelayed(new Runnable() { @Override @@ -116,6 +119,15 @@ public void run() { case "send-feedback": sendFeedback(gleapCallback); break; + case "capture-start": + case "capture-cancel": + case "capture-done": + case "capture-editor": + // Only from the messenger page; nothing is captured before the + // customer taps the SDK's own bar. + GleapCaptureCoordinator.getInstance().onWidgetMessage(activity, command, + gleapCallback.optJSONObject("data")); + break; } } catch (Exception err) { } diff --git a/gleap/src/main/java/io/gleap/GleapWindowCapture.java b/gleap/src/main/java/io/gleap/GleapWindowCapture.java new file mode 100644 index 00000000..13fe0bfa --- /dev/null +++ b/gleap/src/main/java/io/gleap/GleapWindowCapture.java @@ -0,0 +1,819 @@ +package io.gleap; + +import android.annotation.TargetApi; +import android.app.Activity; +import android.graphics.Bitmap; +import android.graphics.Canvas; +import android.graphics.Color; +import android.graphics.Paint; +import android.graphics.Rect; +import android.graphics.RectF; +import android.os.Build; +import android.os.Handler; +import android.os.IBinder; +import android.os.Looper; +import android.os.SystemClock; +import android.text.InputType; +import android.text.method.PasswordTransformationMethod; +import android.view.PixelCopy; +import android.view.SurfaceView; +import android.view.View; +import android.view.ViewGroup; +import android.view.Window; +import android.view.WindowManager; +import android.view.inspector.WindowInspector; +import android.widget.TextView; + +import java.lang.reflect.Field; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.WeakHashMap; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicInteger; + +import io.gleap.callbacks.GetBitmapCallback; + +/** + * Captures what the app shows in the current activity: its window and the windows above it + * that belong to it (dialogs, bottom sheets, popups), in window order. For screenshots and the + * frames of recordings. + *

    + *
  • API 26+: every window with PixelCopy (off the main thread); SurfaceViews (video, maps, + * GL, Flutter) are copied on their own and drawn underneath their window, which only has a + * hole there.
    • + *
  • Below API 26 (or a window that is not hardware accelerated): {@link View#draw} into a + * bitmap on the main thread; SurfaceViews are still copied with PixelCopy on API 24+.
    • + *
  • Windows with FLAG_SECURE, password fields and the views masked with + * {@link Gleap#maskView} are painted black.
    • + *
  • A wrapper SDK that renders its own content (Flutter's GetBitmapCallback) provides the + * picture instead.
    • + *
+ * {@link #capture} must be called on the main thread; the frame is delivered on the given + * handler's thread. + */ +final class GleapWindowCapture { + // Views scanned per window at most (for SurfaceViews and masked views). + private static final int MAX_SCANNED_VIEWS = 10000; + // Masks are a little bigger than the view, against rounding and a scroll between scan and copy. + private static final float MASK_PADDING_PX = 2f; + + // Tests only: copy windows the way API 26-33 does (their Window) on API 34+ as well. + static volatile boolean forceWindowCopy = false; + + // The SDK's own windows (the capture bar, the recording preview): never captured and never + // the window the bar is attached to. Weak, guarded by itself. + private static final Set ownWindows = Collections.newSetFromMap(new WeakHashMap()); + + private static volatile Field decorWindowField; + private static volatile boolean decorWindowFieldLooked; + private static volatile Field globalViewsField; + private static volatile Object windowManagerGlobal; + + private GleapWindowCapture() { + } + + interface Callback { + /** + * @param frame the captured frame, or null when there was nothing to capture + */ + void onFrame(Frame frame); + } + + /** + * One layer of a frame, in frame pixels. + */ + static final class Layer { + static final int DIM = 0; + static final int BITMAP = 1; + static final int BLACK = 2; + + final int type; + final RectF rect; + final float alpha; + // BITMAP only: null while it is copied, or when the copy failed (the layer is left out). + volatile Bitmap bitmap; + // The bitmap came from the app (GetBitmapCallback): never recycled or pooled by the SDK. + boolean external; + + private Layer(int type, RectF rect, float alpha) { + this.type = type; + this.rect = rect; + this.alpha = alpha; + } + } + + /** + * The captured windows as layers, bottom first. Its size is the activity's size times the scale. + */ + static final class Frame { + final int areaWidth; + final int areaHeight; + final float scale; + final int width; + final int height; + final List layers = Collections.synchronizedList(new ArrayList()); + // What capturing cost on the main thread. + volatile long mainThreadMs; + + Frame(int areaWidth, int areaHeight, float scale) { + this.areaWidth = areaWidth; + this.areaHeight = areaHeight; + this.scale = scale; + this.width = Math.max(1, Math.round(areaWidth * scale)); + this.height = Math.max(1, Math.round(areaHeight * scale)); + } + + /** + * Draws the layers at {@code (left, top)}, scaled by {@code drawScale}. + */ + void draw(Canvas canvas, float left, float top, float drawScale) { + Paint bitmapPaint = new Paint(Paint.FILTER_BITMAP_FLAG); + Paint black = new Paint(); + black.setColor(Color.BLACK); + canvas.save(); + canvas.translate(left, top); + canvas.scale(drawScale, drawScale); + canvas.clipRect(0, 0, width, height); + synchronized (layers) { + for (Layer layer : layers) { + if (layer.type == Layer.DIM) { + Paint dim = new Paint(); + dim.setColor(Color.argb(Math.round(Math.max(0f, Math.min(1f, layer.alpha)) * 255), 0, 0, 0)); + canvas.drawRect(0, 0, width, height, dim); + } else if (layer.type == Layer.BLACK) { + canvas.drawRect(layer.rect, black); + } else { + Bitmap bitmap = layer.bitmap; + if (bitmap != null && !bitmap.isRecycled()) { + canvas.drawBitmap(bitmap, null, layer.rect, bitmapPaint); + } + } + } + } + canvas.restore(); + } + + /** + * The frame as one bitmap of {@code width x height}. + */ + Bitmap render() { + Bitmap output = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888); + Canvas canvas = new Canvas(output); + canvas.drawColor(Color.BLACK); + draw(canvas, 0, 0, 1f); + return output; + } + + /** + * Gives the layers' bitmaps back to the pool (or recycles them without one). + */ + void release(BitmapPool pool) { + synchronized (layers) { + for (Layer layer : layers) { + Bitmap bitmap = layer.bitmap; + layer.bitmap = null; + if (bitmap == null || layer.external) { + continue; + } + if (pool != null) { + pool.release(bitmap); + } else { + bitmap.recycle(); + } + } + } + } + + boolean hasContent() { + synchronized (layers) { + for (Layer layer : layers) { + if (layer.type == Layer.BITMAP && layer.bitmap != null) { + return true; + } + } + } + return false; + } + } + + /** + * Reuses the bitmaps of recording frames (they have the same sizes from frame to frame). + */ + static final class BitmapPool { + private static final int MAX_FREE = 8; + private final ArrayList free = new ArrayList<>(); + + synchronized Bitmap obtain(int width, int height) { + for (int i = 0; i < free.size(); i++) { + Bitmap bitmap = free.get(i); + if (bitmap.getWidth() == width && bitmap.getHeight() == height) { + free.remove(i); + return bitmap; + } + } + return Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888); + } + + synchronized void release(Bitmap bitmap) { + if (bitmap == null || bitmap.isRecycled()) { + return; + } + if (free.size() >= MAX_FREE) { + Bitmap oldest = free.remove(0); + oldest.recycle(); + } + free.add(bitmap); + } + + synchronized void clear() { + for (Bitmap bitmap : free) { + bitmap.recycle(); + } + free.clear(); + } + } + + /** + * Counts the copies of a frame that are still running; delivers the frame after the last one. + */ + private static final class Pending { + private final AtomicInteger outstanding = new AtomicInteger(1); + private final Frame frame; + private final Handler handler; + private final Callback callback; + + Pending(Frame frame, Handler handler, Callback callback) { + this.frame = frame; + this.handler = handler; + this.callback = callback; + } + + void add() { + outstanding.incrementAndGet(); + } + + void done() { + if (outstanding.decrementAndGet() == 0) { + deliver(handler, callback, frame); + } + } + } + + /** + * Captures the activity's windows. Main thread only. + * + * @param scale frame pixels per window pixel (1 = full resolution) + * @param excludedRoots the root views of windows to leave out (the SDK's own bar) + * @param resultHandler where the copies finish and the frame is delivered (not the main thread) + * @param pool bitmaps to reuse, or null + */ + static void capture(Activity activity, float scale, Set excludedRoots, Handler resultHandler, + BitmapPool pool, Callback callback) { + long startedAt = SystemClock.uptimeMillis(); + View decor = null; + try { + decor = activity != null && activity.getWindow() != null ? activity.getWindow().peekDecorView() : null; + } catch (Throwable ignore) { + } + if (decor == null || decor.getWidth() <= 0 || decor.getHeight() <= 0 || decor.getWindowToken() == null) { + deliver(resultHandler, callback, null); + return; + } + + int[] origin = new int[2]; + decor.getLocationOnScreen(origin); + Frame frame = new Frame(decor.getWidth(), decor.getHeight(), scale); + + // A wrapper that renders its own content (Flutter) hands over its picture. + Bitmap hostBitmap = hostBitmap(); + if (hostBitmap != null) { + Layer layer = new Layer(Layer.BITMAP, new RectF(0, 0, frame.width, frame.height), 1f); + layer.bitmap = hostBitmap; + layer.external = true; + frame.layers.add(layer); + frame.mainThreadMs = SystemClock.uptimeMillis() - startedAt; + deliver(resultHandler, callback, frame); + return; + } + + Pending pending = new Pending(frame, resultHandler, callback); + try { + Set masked = GleapCapture.maskedViews(); + List roots = windowRoots(activity, decor, excludedRoots); + for (View root : roots) { + addWindow(activity, decor, root, origin, frame, masked, pool, pending, resultHandler); + } + } catch (Throwable error) { + GleapLog.w("Could not capture every window", error); + } + frame.mainThreadMs = SystemClock.uptimeMillis() - startedAt; + pending.done(); + } + + private static void addWindow(Activity activity, View decor, View root, int[] origin, Frame frame, + Set masked, BitmapPool pool, Pending pending, Handler resultHandler) { + WindowManager.LayoutParams params = windowParams(root); + int[] location = new int[2]; + root.getLocationOnScreen(location); + float scale = frame.scale; + RectF windowRect = scaled(location[0] - origin[0], location[1] - origin[1], root.getWidth(), root.getHeight(), scale); + + if (root != decor && params != null && (params.flags & WindowManager.LayoutParams.FLAG_DIM_BEHIND) != 0) { + frame.layers.add(new Layer(Layer.DIM, null, params.dimAmount)); + } + if (params != null && (params.flags & WindowManager.LayoutParams.FLAG_SECURE) != 0) { + // The app keeps this window out of screenshots: so does the SDK. + frame.layers.add(new Layer(Layer.BLACK, windowRect, 1f)); + return; + } + + List surfaces = new ArrayList<>(); + List masks = new ArrayList<>(); + scan(root, masked, surfaces, masks); + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + for (SurfaceView surface : surfaces) { + addSurface(surface, origin, frame, pool, pending, resultHandler); + } + } + + Layer windowLayer = new Layer(Layer.BITMAP, windowRect, 1f); + frame.layers.add(windowLayer); + copyWindow(activity, decor, root, windowLayer, pool, pending, resultHandler); + + Rect visible = new Rect(); + for (View view : masks) { + try { + if (!view.getGlobalVisibleRect(visible)) { + continue; + } + // Window coordinates → the activity's. + RectF mask = scaled(visible.left + location[0] - origin[0], visible.top + location[1] - origin[1], + visible.width(), visible.height(), scale); + mask.inset(-MASK_PADDING_PX, -MASK_PADDING_PX); + frame.layers.add(new Layer(Layer.BLACK, mask, 1f)); + } catch (Throwable ignore) { + } + } + } + + private static RectF scaled(float left, float top, float width, float height, float scale) { + return new RectF(left * scale, top * scale, (left + width) * scale, (top + height) * scale); + } + + private static int scaledSize(int size, float scale) { + return Math.max(1, Math.round(size * scale)); + } + + // --------------------------------------------------------------------------------------------- + // Copying + + private static void copyWindow(final Activity activity, final View decor, final View root, final Layer layer, + final BitmapPool pool, final Pending pending, final Handler resultHandler) { + final float scale = layer.rect.width() / Math.max(1, root.getWidth()); + final Bitmap bitmap = obtain(pool, scaledSize(root.getWidth(), scale), scaledSize(root.getHeight(), scale)); + if (bitmap == null) { + return; + } + boolean hardware = root.isHardwareAccelerated(); + boolean started = false; + try { + CopyResult result = new CopyResult() { + @Override + public void onResult(boolean success) { + finishWindowCopy(root, layer, bitmap, scale, success, pool, pending); + } + }; + if (Build.VERSION.SDK_INT >= 34 && hardware && !forceWindowCopy) { + pending.add(); + started = true; + requestWindowCopy(root, bitmap, result, resultHandler); + return; + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && hardware) { + Window window = windowOf(activity, decor, root); + if (window != null) { + pending.add(); + started = true; + requestWindowCopy(window, bitmap, result, resultHandler); + return; + } + } + } catch (Throwable error) { + // E.g. the window has no surface (anymore): draw it instead. The copy did not start. + if (started) { + pending.done(); + } + } + drawWindow(root, layer, bitmap, scale, pool); + } + + interface CopyResult { + void onResult(boolean success); + } + + @TargetApi(34) + private static void requestWindowCopy(View root, Bitmap bitmap, final CopyResult result, final Handler handler) { + // The window's own area: without a source rect ofWindow(View) takes the surface insets + // (a dialog's shadow) as the source rect, and the whole surface is copied, scaled down. + PixelCopy.Request request = PixelCopy.Request.Builder.ofWindow(root) + .setSourceRect(new Rect(0, 0, root.getWidth(), root.getHeight())) + .setDestinationBitmap(bitmap) + .build(); + PixelCopy.request(request, executor(handler), new java.util.function.Consumer() { + @Override + public void accept(PixelCopy.Result copy) { + result.onResult(copy.getStatus() == PixelCopy.SUCCESS); + } + }); + } + + @TargetApi(Build.VERSION_CODES.O) + private static void requestWindowCopy(Window window, Bitmap bitmap, final CopyResult result, Handler handler) { + PixelCopy.request(window, bitmap, new PixelCopy.OnPixelCopyFinishedListener() { + @Override + public void onPixelCopyFinished(int copyResult) { + result.onResult(copyResult == PixelCopy.SUCCESS); + } + }, handler); + } + + // On the result thread: a failed copy is drawn on the main thread instead. + private static void finishWindowCopy(final View root, final Layer layer, final Bitmap bitmap, final float scale, + boolean success, final BitmapPool pool, final Pending pending) { + if (success) { + layer.bitmap = bitmap; + pending.done(); + return; + } + GleapMainThread.post(new Runnable() { + @Override + public void run() { + try { + drawWindow(root, layer, bitmap, scale, pool); + } finally { + pending.done(); + } + } + }); + } + + /** + * Draws the window's views into the bitmap (main thread). Views that cannot be drawn in + * software (e.g. hardware bitmaps) leave the window out. + */ + private static void drawWindow(View root, Layer layer, Bitmap bitmap, float scale, BitmapPool pool) { + try { + bitmap.eraseColor(Color.TRANSPARENT); + Canvas canvas = new Canvas(bitmap); + canvas.scale(bitmap.getWidth() / (float) Math.max(1, root.getWidth()), + bitmap.getHeight() / (float) Math.max(1, root.getHeight())); + root.draw(canvas); + layer.bitmap = bitmap; + } catch (Throwable error) { + GleapLog.w("Could not draw a window for the capture", error); + release(pool, bitmap); + } + } + + @TargetApi(Build.VERSION_CODES.N) + private static void addSurface(SurfaceView surface, int[] origin, Frame frame, final BitmapPool pool, + final Pending pending, Handler resultHandler) { + try { + if (surface.getWidth() <= 0 || surface.getHeight() <= 0 || !surface.getHolder().getSurface().isValid()) { + return; + } + int[] location = new int[2]; + surface.getLocationOnScreen(location); + final Layer layer = new Layer(Layer.BITMAP, + scaled(location[0] - origin[0], location[1] - origin[1], surface.getWidth(), surface.getHeight(), frame.scale), 1f); + final Bitmap bitmap = obtain(pool, scaledSize(surface.getWidth(), frame.scale), scaledSize(surface.getHeight(), frame.scale)); + if (bitmap == null) { + return; + } + // Below the window: the window only has a hole where the surface shows. + frame.layers.add(layer); + pending.add(); + try { + PixelCopy.request(surface, bitmap, new PixelCopy.OnPixelCopyFinishedListener() { + @Override + public void onPixelCopyFinished(int copyResult) { + if (copyResult == PixelCopy.SUCCESS) { + layer.bitmap = bitmap; + } else { + release(pool, bitmap); + } + pending.done(); + } + }, resultHandler); + } catch (Throwable error) { + release(pool, bitmap); + pending.done(); + } + } catch (Throwable ignore) { + } + } + + private static Bitmap obtain(BitmapPool pool, int width, int height) { + try { + return pool != null ? pool.obtain(width, height) : Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888); + } catch (Throwable error) { + // Out of memory: the window is left out. + GleapLog.w("No memory for a capture bitmap", error); + return null; + } + } + + private static void release(BitmapPool pool, Bitmap bitmap) { + if (pool != null) { + pool.release(bitmap); + } else if (bitmap != null) { + bitmap.recycle(); + } + } + + private static Executor executor(final Handler handler) { + return new Executor() { + @Override + public void execute(Runnable command) { + handler.post(command); + } + }; + } + + private static void deliver(Handler handler, final Callback callback, final Frame frame) { + Runnable delivery = new Runnable() { + @Override + public void run() { + try { + callback.onFrame(frame); + } catch (Throwable error) { + GleapLog.w("Could not handle a captured frame", error); + } + } + }; + if (handler != null) { + handler.post(delivery); + } else { + new Handler(Looper.getMainLooper()).post(delivery); + } + } + + private static Bitmap hostBitmap() { + try { + GetBitmapCallback bitmapCallback = GleapCallbacks.getInstance().getGetBitmapCallback(); + if (bitmapCallback == null) { + return null; + } + Bitmap bitmap = bitmapCallback.getBitmap(); + return bitmap != null && !bitmap.isRecycled() && bitmap.getWidth() > 0 ? bitmap : null; + } catch (Throwable error) { + GleapLog.w("The app's bitmap callback failed", error); + return null; + } + } + + // --------------------------------------------------------------------------------------------- + // Windows + + /** + * The windows of the activity to capture, bottom first: the activity's window, then its + * dialogs in the order they were shown, each followed by its panels (popups, menus). + */ + static List windowRoots(Activity activity, View decor, Set excludedRoots) { + List ordered = new ArrayList<>(); + ordered.add(decor); + List all = globalWindowViews(); + if (all.isEmpty()) { + return ordered; + } + + IBinder appToken = null; + try { + appToken = activity.getWindow().getAttributes().token; + } catch (Throwable ignore) { + } + + List topLevel = new ArrayList<>(); + topLevel.add(decor); + for (View view : all) { + if (view == decor || !isCapturable(view, excludedRoots)) { + continue; + } + WindowManager.LayoutParams params = windowParams(view); + if (params == null || appToken == null || params.token != appToken) { + continue; + } + if (params.type > WindowManager.LayoutParams.FIRST_APPLICATION_WINDOW + && params.type <= WindowManager.LayoutParams.LAST_APPLICATION_WINDOW + && params.type != WindowManager.LayoutParams.TYPE_APPLICATION_STARTING) { + topLevel.add(view); + } + } + + ordered.clear(); + for (View window : topLevel) { + ordered.add(window); + IBinder parentToken = window.getWindowToken(); + if (parentToken == null) { + continue; + } + for (View view : all) { + if (view == window || !isCapturable(view, excludedRoots)) { + continue; + } + WindowManager.LayoutParams params = windowParams(view); + if (params != null && params.token == parentToken + && params.type >= WindowManager.LayoutParams.FIRST_SUB_WINDOW + && params.type <= WindowManager.LayoutParams.LAST_SUB_WINDOW) { + ordered.add(view); + } + } + } + return ordered; + } + + /** + * The window the SDK's bar belongs on: the topmost dialog of the activity, else the + * activity's own window. + */ + static View topWindow(Activity activity, View decor, Set excludedRoots) { + View top = decor; + IBinder appToken; + try { + appToken = activity.getWindow().getAttributes().token; + } catch (Throwable error) { + return decor; + } + for (View view : globalWindowViews()) { + if (view == decor || !isCapturable(view, excludedRoots)) { + continue; + } + WindowManager.LayoutParams params = windowParams(view); + if (params != null && appToken != null && params.token == appToken + && params.type > WindowManager.LayoutParams.FIRST_APPLICATION_WINDOW + && params.type <= WindowManager.LayoutParams.LAST_APPLICATION_WINDOW + && params.type != WindowManager.LayoutParams.TYPE_APPLICATION_STARTING + && view.getWindowToken() != null) { + top = view; + } + } + return top; + } + + static void addOwnWindow(View root) { + if (root != null) { + synchronized (ownWindows) { + ownWindows.add(root); + } + } + } + + static void removeOwnWindow(View root) { + if (root != null) { + synchronized (ownWindows) { + ownWindows.remove(root); + } + } + } + + private static boolean isOwnWindow(View root) { + synchronized (ownWindows) { + return ownWindows.contains(root); + } + } + + private static boolean isCapturable(View view, Set excludedRoots) { + return view != null + && !isOwnWindow(view) + && (excludedRoots == null || !excludedRoots.contains(view)) + && view.getWindowToken() != null + && view.getVisibility() == View.VISIBLE + && view.getWidth() > 0 + && view.getHeight() > 0; + } + + static WindowManager.LayoutParams windowParams(View root) { + ViewGroup.LayoutParams params = root.getLayoutParams(); + return params instanceof WindowManager.LayoutParams ? (WindowManager.LayoutParams) params : null; + } + + /** + * The root views of all windows of the process (WindowInspector on API 29+, WindowManagerGlobal + * below); empty when they cannot be read. + */ + @SuppressWarnings("unchecked") + static List globalWindowViews() { + try { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + return new ArrayList<>(WindowInspector.getGlobalWindowViews()); + } + Field field = globalViewsField; + Object global = windowManagerGlobal; + if (field == null || global == null) { + Class globalClass = Class.forName("android.view.WindowManagerGlobal"); + global = globalClass.getMethod("getInstance").invoke(null); + field = globalClass.getDeclaredField("mViews"); + field.setAccessible(true); + windowManagerGlobal = global; + globalViewsField = field; + } + Object views = field.get(global); + if (views instanceof List) { + return new ArrayList<>((List) views); + } + } catch (Throwable ignore) { + } + return new ArrayList<>(); + } + + /** + * The Window of a root view: the activity's, or a dialog's (its DecorView knows it). + */ + private static Window windowOf(Activity activity, View decor, View root) { + if (root == decor) { + return activity.getWindow(); + } + try { + if (!decorWindowFieldLooked) { + Field field = null; + for (Class type = root.getClass(); type != null && type != Object.class; type = type.getSuperclass()) { + if (type.getName().endsWith("DecorView")) { + field = type.getDeclaredField("mWindow"); + field.setAccessible(true); + break; + } + } + decorWindowField = field; + decorWindowFieldLooked = field != null; + } + Field field = decorWindowField; + if (field != null && field.getDeclaringClass().isInstance(root)) { + Object window = field.get(root); + return window instanceof Window ? (Window) window : null; + } + } catch (Throwable ignore) { + } + return null; + } + + /** + * Finds the SurfaceViews and the views to paint black (password fields, masked views) in a + * window. Hidden subtrees are skipped; a masked view's children are covered by its mask. + */ + private static void scan(View root, Set masked, List surfaces, List masks) { + ArrayDeque stack = new ArrayDeque<>(); + stack.push(root); + int budget = MAX_SCANNED_VIEWS; + while (!stack.isEmpty() && budget-- > 0) { + View view = stack.pop(); + if (view.getVisibility() != View.VISIBLE) { + continue; + } + if (masked.contains(view) || isSecureText(view)) { + masks.add(view); + continue; + } + if (view instanceof SurfaceView) { + surfaces.add((SurfaceView) view); + } + if (view instanceof ViewGroup) { + ViewGroup group = (ViewGroup) view; + for (int i = group.getChildCount() - 1; i >= 0; i--) { + View child = group.getChildAt(i); + if (child != null) { + stack.push(child); + } + } + } + } + } + + /** + * Password fields (text, number and web passwords, also while the password is shown). + */ + static boolean isSecureText(View view) { + if (!(view instanceof TextView)) { + return false; + } + TextView text = (TextView) view; + if (isPasswordInputType(text.getInputType())) { + return true; + } + return text.getTransformationMethod() instanceof PasswordTransformationMethod; + } + + static boolean isPasswordInputType(int inputType) { + int inputClass = inputType & InputType.TYPE_MASK_CLASS; + int variation = inputType & InputType.TYPE_MASK_VARIATION; + if (inputClass == InputType.TYPE_CLASS_TEXT) { + return variation == InputType.TYPE_TEXT_VARIATION_PASSWORD + || variation == InputType.TYPE_TEXT_VARIATION_VISIBLE_PASSWORD + || variation == InputType.TYPE_TEXT_VARIATION_WEB_PASSWORD; + } + return inputClass == InputType.TYPE_CLASS_NUMBER && variation == InputType.TYPE_NUMBER_VARIATION_PASSWORD; + } +} diff --git a/gleap/src/test/java/io/gleap/GleapCaptureGeometryTest.java b/gleap/src/test/java/io/gleap/GleapCaptureGeometryTest.java new file mode 100644 index 00000000..9c9172b1 --- /dev/null +++ b/gleap/src/test/java/io/gleap/GleapCaptureGeometryTest.java @@ -0,0 +1,104 @@ +package io.gleap; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +/** + * The sizes of screenshots and recordings: the long edge limits, the even (aligned) encoder sizes + * H.264 needs, the letterbox after a rotation, and the frame rate limits. + */ +public class GleapCaptureGeometryTest { + @Test + public void screenshotsKeepTheirSizeUpToTheLimitAndShrinkAboveIt() { + assertArrayEquals(new int[]{1080, 2400}, GleapCaptureGeometry.fitLongEdge(1080, 2400, 2560)); + assertArrayEquals(new int[]{2560, 1600}, GleapCaptureGeometry.fitLongEdge(2560, 1600, 2560)); + // A 12.9" tablet: 2732 x 2048 → long edge 2560. + assertArrayEquals(new int[]{2560, 1919}, GleapCaptureGeometry.fitLongEdge(2732, 2048, 2560)); + assertArrayEquals(new int[]{1, 1}, GleapCaptureGeometry.fitLongEdge(0, 0, 2560)); + } + + @Test + public void encoderSizesAreEvenAlignedAndAtMost1280() { + int[] phone = GleapCaptureGeometry.encoderSize(1080, 2400, 1280, 2); + assertArrayEquals(new int[]{576, 1280}, phone); + + // Odd results are rounded down to even sizes. + int[] odd = GleapCaptureGeometry.encoderSize(1081, 2399, 1280, 2); + assertEquals(0, odd[0] % 2); + assertEquals(0, odd[1] % 2); + assertTrue(odd[1] <= 1280); + + // An encoder that needs multiples of 16. + int[] aligned = GleapCaptureGeometry.encoderSize(1080, 2400, 1280, 16); + assertArrayEquals(new int[]{576, 1280}, aligned); + int[] tablet = GleapCaptureGeometry.encoderSize(2560, 1600, 1280, 16); + assertArrayEquals(new int[]{1280, 800}, tablet); + int[] foldable = GleapCaptureGeometry.encoderSize(2208, 1840, 1280, 16); + assertEquals(0, foldable[0] % 16); + assertEquals(0, foldable[1] % 16); + assertTrue(foldable[0] <= 1280 && foldable[1] <= 1280); + + // Small windows are never upscaled and never 0. + assertArrayEquals(new int[]{640, 360}, GleapCaptureGeometry.encoderSize(640, 360, 1280, 2)); + assertArrayEquals(new int[]{16, 16}, GleapCaptureGeometry.encoderSize(3, 3, 1280, 16)); + } + + @Test + public void aRotatedFrameIsLetterboxedIntoTheFixedOutput() { + // Portrait recording (576 x 1280), the phone turned to landscape (2400 x 1080). + float[] box = GleapCaptureGeometry.letterbox(2400, 1080, 576, 1280); + assertEquals(576f, box[2], 0.01f); + assertEquals(259.2f, box[3], 0.01f); + assertEquals(0f, box[0], 0.01f); + // Centered vertically. + assertEquals((1280 - 259.2f) / 2, box[1], 0.01f); + + // Same aspect: the frame fills the output. + float[] same = GleapCaptureGeometry.letterbox(1080, 2400, 576, 1280); + assertArrayEquals(new float[]{0f, 0f, 576f, 1280f}, same, 0.01f); + + // A narrower window (multi-window) gets bars left and right. + float[] narrow = GleapCaptureGeometry.letterbox(540, 2400, 576, 1280); + assertEquals(1280f, narrow[3], 0.01f); + assertEquals((576 - narrow[2]) / 2, narrow[0], 0.01f); + } + + @Test + public void theFrameRateStaysBetweenTwoAndEightFramesASecond() { + long interval = GleapCaptureGeometry.TARGET_FRAME_INTERVAL_MS; + // Expensive frames (more than 30 % of the interval on the main thread) slow down to 2 fps. + for (int i = 0; i < 10; i++) { + interval = GleapCaptureGeometry.nextFrameInterval(interval, 200, true); + } + assertEquals(GleapCaptureGeometry.MAX_FRAME_INTERVAL_MS, interval); + + // Frames that do not keep up slow down as well. + assertTrue(GleapCaptureGeometry.nextFrameInterval(250, 10, false) > 250); + + // Cheap frames speed up to at most 8 fps. + interval = GleapCaptureGeometry.TARGET_FRAME_INTERVAL_MS; + for (int i = 0; i < 20; i++) { + interval = GleapCaptureGeometry.nextFrameInterval(interval, 1, true); + } + assertEquals(GleapCaptureGeometry.MIN_FRAME_INTERVAL_MS, interval); + + // Moderate cost after a slow phase: back towards 4 fps. + assertTrue(GleapCaptureGeometry.nextFrameInterval(500, 60, true) < 500); + assertEquals(250, GleapCaptureGeometry.nextFrameInterval(250, 40, true)); + } + + @Test + public void theTimerAndTheDurationLimit() { + assertEquals("00:00", GleapCaptureGeometry.formatClock(0)); + assertEquals("01:05", GleapCaptureGeometry.formatClock(65)); + assertEquals("03:00", GleapCaptureGeometry.formatClock(180)); + + assertEquals(60, GleapCaptureGeometry.clampDuration(0)); + assertEquals(5, GleapCaptureGeometry.clampDuration(1)); + assertEquals(180, GleapCaptureGeometry.clampDuration(600)); + assertEquals(45, GleapCaptureGeometry.clampDuration(45)); + } +} diff --git a/gleap/src/test/java/io/gleap/GleapCaptureTest.java b/gleap/src/test/java/io/gleap/GleapCaptureTest.java new file mode 100644 index 00000000..afc1eccc --- /dev/null +++ b/gleap/src/test/java/io/gleap/GleapCaptureTest.java @@ -0,0 +1,115 @@ +package io.gleap; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.json.JSONObject; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; + +/** + * Who may start a capture and what the SDK announces: capture commands are only taken from the + * messenger's own origin, request ids never leave the id format, and the capabilities follow the + * app's switches. + */ +public class GleapCaptureTest { + private static final String WIDGET = "https://messenger-app.gleap.io/appnew"; + + @Test + public void onlyTheMessengerOriginMayStartACapture() { + assertTrue(GleapCapture.sameOrigin("https://messenger-app.gleap.io/appnew?lang=en&gleapId=1", WIDGET)); + assertTrue(GleapCapture.sameOrigin("https://MESSENGER-APP.gleap.io:443/other", WIDGET)); + + assertFalse(GleapCapture.sameOrigin("http://messenger-app.gleap.io/appnew", WIDGET)); + assertFalse(GleapCapture.sameOrigin("https://messenger-app.gleap.io.evil.com/appnew", WIDGET)); + assertFalse(GleapCapture.sameOrigin("https://evil.com/?u=https://messenger-app.gleap.io/appnew", WIDGET)); + assertFalse(GleapCapture.sameOrigin("https://messenger-app.gleap.io@evil.com/appnew", WIDGET)); + assertFalse(GleapCapture.sameOrigin("https://messenger-app.gleap.io:8443/appnew", WIDGET)); + assertFalse(GleapCapture.sameOrigin("javascript:alert(1)", WIDGET)); + assertFalse(GleapCapture.sameOrigin("file:///android_asset/appnew.html", WIDGET)); + assertFalse(GleapCapture.sameOrigin("about:blank", WIDGET)); + assertFalse(GleapCapture.sameOrigin(null, WIDGET)); + assertFalse(GleapCapture.sameOrigin(WIDGET, null)); + + // A custom frame url (setFrameUrl), e.g. a local server. + assertTrue(GleapCapture.sameOrigin("http://10.0.2.2:8787/widget/appnew?x=1", "http://10.0.2.2:8787/widget/appnew")); + assertFalse(GleapCapture.sameOrigin("http://10.0.2.2:8788/widget/appnew", "http://10.0.2.2:8787/widget/appnew")); + } + + @Test + public void requestIdsStayInsideTheirPath() { + assertTrue(GleapCapture.isValidRequestId("65f0c0ffee0123456789abcd")); + assertFalse(GleapCapture.isValidRequestId("../../bugs/v2")); + assertFalse(GleapCapture.isValidRequestId("id/claim")); + assertFalse(GleapCapture.isValidRequestId("id?x=1")); + assertFalse(GleapCapture.isValidRequestId("")); + assertFalse(GleapCapture.isValidRequestId(null)); + assertEquals("/v3/shared/capture-requests/abc/logs", GleapCaptureApi.requestPath("abc", "logs")); + try { + GleapCaptureApi.requestPath("a/b", "logs"); + throw new AssertionError("An invalid id was accepted"); + } catch (IllegalArgumentException expected) { + } + } + + @Test + public void theCapabilitiesFollowTheAppsSwitches() throws Exception { + assertEquals(Arrays.asList("capture.screenshot", "capture.recording", "capture.logs"), + GleapCapture.caps(true, true, true)); + // Below API 26: no recordings. + assertEquals(Arrays.asList("capture.screenshot", "capture.logs"), GleapCapture.caps(true, false, true)); + assertEquals(Collections.singletonList("capture.logs"), GleapCapture.caps(false, true, true)); + assertEquals(Collections.emptyList(), GleapCapture.caps(false, true, false)); + assertEquals("capture.screenshot,capture.logs", + GleapCapture.capsQueryValue(GleapCapture.caps(true, false, true))); + + JSONObject enabled = GleapCapture.capabilities(true, true, "FLUTTER", "19.0.0"); + assertEquals(1, enabled.getInt("version")); + assertEquals("android", enabled.getString("platform")); + assertEquals("FLUTTER", enabled.getString("sdkType")); + assertTrue(enabled.getBoolean("screenshot")); + assertTrue(enabled.getBoolean("recording")); + assertEquals("frames", enabled.getString("recordingMethod")); + assertFalse(enabled.getBoolean("microphone")); + + // setCaptureEnabled(false): nothing is offered, the widget falls back to the upload. + JSONObject disabled = GleapCapture.capabilities(false, true, "NATIVE", "19.0.0"); + assertFalse(disabled.getBoolean("screenshot")); + assertFalse(disabled.getBoolean("recording")); + assertFalse(disabled.has("recordingMethod")); + + JSONObject oldDevice = GleapCapture.capabilities(true, false, "NATIVE", "19.0.0"); + assertTrue(oldDevice.getBoolean("screenshot")); + assertFalse(oldDevice.getBoolean("recording")); + } + + @Test + public void aCaptureStartIsReadWithItsLimitsAndLabels() throws Exception { + JSONObject start = new JSONObject() + .put("requestId", "65f0c0ffee0123456789abcd") + .put("kind", "recording") + .put("ticketShareToken", "share-1") + .put("options", new JSONObject().put("maxDurationSec", 900).put("attachLogs", false)) + .put("labels", new JSONObject().put("barStop", "Stopp").put("barStart", " ")); + + GleapCaptureRequest request = GleapCaptureRequest.fromStart(start); + + assertEquals("recording", request.kind); + assertTrue(request.isRecording()); + assertEquals("share-1", request.ticketShareToken); + assertEquals(180, request.maxDurationSec); + assertFalse(request.attachLogs); + assertEquals("Stopp", request.label("barStop")); + // Blank or missing labels fall back to English. + assertEquals("Start recording", request.label("barStart")); + assertEquals("Cancel", request.label("barCancel")); + + assertNull(GleapCaptureRequest.fromStart(new JSONObject(start.toString()).put("kind", "any"))); + assertNull(GleapCaptureRequest.fromStart(new JSONObject(start.toString()).put("requestId", "../x"))); + assertNull(GleapCaptureRequest.fromStart(null)); + } +} From aeb0ebe4a9493b96f0587a62a94fb4ae37e7ceae Mon Sep 17 00:00:00 2001 From: Lukas Boehler Date: Wed, 30 Sep 2026 21:42:59 -0400 Subject: [PATCH 05/25] feat: capture demo screen and local stub mode in the example app A "Capture Demo" screen to try capture requests on: a GLSurfaceView (SurfaceView content), a password field and a view masked with Gleap.maskView, a dialog with a password field, a running clock, a way to open the same screen again, and switches for setCaptureEnabled and setRemoteLogCollectionEnabled. The app registers a log flush handler the way a wrapper SDK would. Debug builds only: with adb shell setprop debug.gleap.stub http://10.0.2.2:8787 the app sends every request (API, WebSocket, widget page, banners, modals) to a local stub server instead of Gleap, with a fake SDK key; cleartext is allowed for 10.0.2.2 / localhost in debug builds. The error callback prints the stack trace. Co-Authored-By: Claude Opus 5.5 --- app/src/debug/AndroidManifest.xml | 7 + .../gleap_stub_network_security_config.xml | 11 ++ app/src/main/AndroidManifest.xml | 4 + .../io/gleap_android_sdk/CaptureDemo.java | 143 ++++++++++++++++++ .../io/gleap_android_sdk/MainActivity.java | 7 + .../io/gleap_android_sdk/MainApplication.java | 44 +++++- .../main/res/layout/activity_capture_demo.xml | 104 +++++++++++++ app/src/main/res/layout/activity_main.xml | 7 + 8 files changed, 326 insertions(+), 1 deletion(-) create mode 100644 app/src/debug/AndroidManifest.xml create mode 100644 app/src/debug/res/xml/gleap_stub_network_security_config.xml create mode 100644 app/src/main/java/gleap/io/gleap_android_sdk/CaptureDemo.java create mode 100644 app/src/main/res/layout/activity_capture_demo.xml diff --git a/app/src/debug/AndroidManifest.xml b/app/src/debug/AndroidManifest.xml new file mode 100644 index 00000000..dc525375 --- /dev/null +++ b/app/src/debug/AndroidManifest.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/app/src/debug/res/xml/gleap_stub_network_security_config.xml b/app/src/debug/res/xml/gleap_stub_network_security_config.xml new file mode 100644 index 00000000..6503c516 --- /dev/null +++ b/app/src/debug/res/xml/gleap_stub_network_security_config.xml @@ -0,0 +1,11 @@ + + + + + + 10.0.2.2 + 127.0.0.1 + localhost + + diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index f1d47fb6..3d9066b5 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -62,6 +62,10 @@ android:exported="false" android:launchMode="singleInstance" android:theme="@style/AppTheme.NoActionBar" /> + Gleap.getInstance().open()); + findViewById(R.id.capture_dialog).setOnClickListener(view -> showDialog()); + findViewById(R.id.capture_next).setOnClickListener(view -> + startActivity(new Intent(CaptureDemo.this, CaptureDemo.class))); + findViewById(R.id.capture_log).setOnClickListener(view -> { + Gleap.getInstance().log("Capture demo: a log line"); + Gleap.getInstance().trackEvent("capture-demo-tap"); + }); + + TextView captureSwitch = findViewById(R.id.capture_switch); + captureSwitch.setOnClickListener(view -> { + captureEnabled = !captureEnabled; + Gleap.getInstance().setCaptureEnabled(captureEnabled); + captureSwitch.setText(captureEnabled ? "Screen capture: on" : "Screen capture: off"); + }); + TextView logsSwitch = findViewById(R.id.capture_logs_switch); + logsSwitch.setOnClickListener(view -> { + remoteLogs = !remoteLogs; + Gleap.getInstance().setRemoteLogCollectionEnabled(remoteLogs); + logsSwitch.setText(remoteLogs ? "Remote logs: on" : "Remote logs: off"); + }); + } + + // Process-wide like the SDK's switches (a new screen starts with them on again). + private static boolean captureEnabled = true; + private static boolean remoteLogs = true; + + private void showDialog() { + EditText secret = new EditText(this); + secret.setInputType(InputType.TYPE_CLASS_TEXT | InputType.TYPE_TEXT_VARIATION_PASSWORD); + secret.setText("dialog-secret"); + new AlertDialog.Builder(this) + .setTitle("A dialog to capture") + .setMessage("The capture bar moves above this dialog. Its password field is painted black.") + .setView(secret) + .setPositiveButton("Close", null) + .show(); + } + + @Override + protected void onResume() { + super.onResume(); + glView.onResume(); + handler.post(tick); + } + + @Override + protected void onPause() { + handler.removeCallbacks(tick); + glView.onPause(); + super.onPause(); + } + + /** + * A changing background with a yellow square moving left and right. + */ + private static final class MovingSquareRenderer implements GLSurfaceView.Renderer { + private final long start = SystemClock.uptimeMillis(); + private int width; + private int height; + + @Override + public void onSurfaceCreated(GL10 gl, EGLConfig config) { + } + + @Override + public void onSurfaceChanged(GL10 gl, int width, int height) { + this.width = width; + this.height = height; + GLES20.glViewport(0, 0, width, height); + } + + @Override + public void onDrawFrame(GL10 gl) { + float t = (SystemClock.uptimeMillis() - start) / 1000f; + GLES20.glDisable(GLES20.GL_SCISSOR_TEST); + GLES20.glClearColor(0.1f, 0.45f + 0.35f * (float) Math.sin(t), 0.35f, 1f); + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + int size = Math.max(1, height / 2); + int x = (int) ((width - size) * (0.5 + 0.5 * Math.sin(t * 1.3))); + GLES20.glEnable(GLES20.GL_SCISSOR_TEST); + GLES20.glScissor(x, height / 4, size, size); + GLES20.glClearColor(1f, 0.85f, 0f, 1f); + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + GLES20.glDisable(GLES20.GL_SCISSOR_TEST); + } + } +} diff --git a/app/src/main/java/gleap/io/gleap_android_sdk/MainActivity.java b/app/src/main/java/gleap/io/gleap_android_sdk/MainActivity.java index 2dc813eb..0bad3027 100644 --- a/app/src/main/java/gleap/io/gleap_android_sdk/MainActivity.java +++ b/app/src/main/java/gleap/io/gleap_android_sdk/MainActivity.java @@ -159,6 +159,13 @@ public void onClick(View view) { }); + findViewById(R.id.capture).setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View view) { + startActivity(new Intent(MainActivity.this, CaptureDemo.class)); + } + }); + findViewById(R.id.network).setOnClickListener(new View.OnClickListener() { @Override public void onClick(View view) { diff --git a/app/src/main/java/gleap/io/gleap_android_sdk/MainApplication.java b/app/src/main/java/gleap/io/gleap_android_sdk/MainApplication.java index e65c40d5..d888e052 100644 --- a/app/src/main/java/gleap/io/gleap_android_sdk/MainApplication.java +++ b/app/src/main/java/gleap/io/gleap_android_sdk/MainApplication.java @@ -18,6 +18,18 @@ import io.gleap.callbacks.UnRegisterPushMessageGroupCallback; public class MainApplication extends Application { + private static String systemProperty(String key) { + try { + Process process = new ProcessBuilder("getprop", key).start(); + java.io.BufferedReader reader = new java.io.BufferedReader(new java.io.InputStreamReader(process.getInputStream())); + String value = reader.readLine(); + reader.close(); + return value != null ? value.trim() : ""; + } catch (Exception e) { + return ""; + } + } + @Override public void onCreate() { super.onCreate(); @@ -27,10 +39,40 @@ public void onCreate() { @Override public void onError(Throwable error, String context) { System.out.println(context); + if (error != null) { + error.printStackTrace(); + } } }); - Gleap.initialize("ogWhNhuiZcGWrva5nlDS8l7a78OfaLlV", this); + String sdkKey = "ogWhNhuiZcGWrva5nlDS8l7a78OfaLlV"; + // Debug builds only: every request goes to a local stub server instead of Gleap, e.g. + // adb shell setprop debug.gleap.stub http://10.0.2.2:8787 (then restart the app). + String stub = BuildConfig.DEBUG ? systemProperty("debug.gleap.stub") : ""; + if (stub.startsWith("http://") || stub.startsWith("https://")) { + Gleap.getInstance().setApiUrl(stub); + Gleap.getInstance().setWSApiUrl(stub.replaceFirst("^http", "ws") + "/ws"); + Gleap.getInstance().setFrameUrl(stub + "/widget/appnew"); + Gleap.getInstance().setBannerUrl(stub + "/outbound"); + Gleap.getInstance().setModalUrl(stub + "/outbound/modal"); + Gleap.getInstance().setRealtimeHost(android.net.Uri.parse(stub).getAuthority()); + sdkKey = "stub-sdk-key"; + } + + Gleap.initialize(sdkKey, this); + + // What a wrapper SDK does: hand over its buffered logs before a capture request's logs + // are collected, then call done. + Gleap.getInstance().setLogFlushHandler(done -> { + try { + Gleap.getInstance().attachConsoleLogs(new org.json.JSONArray().put(new JSONObject() + .put("date", "2026-09-30T00:00:00.000Z") + .put("priority", "INFO") + .put("log", "Flushed by the app's log flush handler"))); + } catch (JSONException ignore) { + } + done.run(); + }); Gleap.getInstance().setTags(new String[] { "Android", "Tags", diff --git a/app/src/main/res/layout/activity_capture_demo.xml b/app/src/main/res/layout/activity_capture_demo.xml new file mode 100644 index 00000000..e67395a2 --- /dev/null +++ b/app/src/main/res/layout/activity_capture_demo.xml @@ -0,0 +1,104 @@ + + + + + + + + + + + + + + + + + + + +