diff --git a/.gitignore b/.gitignore index b85b08c7..ad5f133c 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,5 @@ local.properties /compile_commands.json /tmp video.sdp +ISSUE.md +/detect_model diff --git a/README.md b/README.md index ae168c40..b4aac9f2 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,22 @@ Higher values → leaner bandwidth, less resilience. --- +## Object Detection (Optional) +PixelPilot features an optional, real-time object detection overlay using Google's MediaPipe Object Detection library. + +### Features +* **Live Overlay**: Draws bounding boxes, class names, and confidence scores directly over the standard video stream. +* **Hardware Acceleration**: Toggle between CPU and GPU delegates to optimize performance for your device. +* **Model Choices**: Supports switching between `EfficientDet-Lite0` (lightweight/faster) and `EfficientDet-Lite2` (larger/more accurate) models. +* **Automated Setup**: The necessary TFLite model files are downloaded automatically from Google's CDN during the Gradle build phase. + +### How to Use +1. In the application settings menu, click on the **Object Detection** submenu. +2. Select **Enable** to toggle the detection overlay on/off. +3. Choose your preferred hardware **Delegate** (CPU/GPU) and **Model** (Lite0/Lite2) based on your device's processing capabilities. + +--- + ## List of potential improvements: * adaptive link [x] * 40 MHz bandwidth [?] - works but buggy diff --git a/app/build.gradle b/app/build.gradle index 3bb89146..66bbd35b 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -12,6 +12,11 @@ android { targetSdk = 34 versionCode = 1 versionName = "0.23.0" + + ndk { + abiFilters.add("arm64-v8a") + abiFilters.add("armeabi-v7a") + } } signingConfigs { @@ -40,6 +45,12 @@ android { buildFeatures { viewBinding = true } + packaging { + jniLibs { + excludes += ["lib/armeabi-v7a/libmediapipe_tasks_vision_jni.so"] + useLegacyPackaging = true + } + } buildToolsVersion '35.0.0' ndkVersion '26.1.10909125' } @@ -52,4 +63,5 @@ dependencies { implementation(libs.appcompat) implementation(libs.material) implementation(libs.philjay.mpandroidchart) + implementation(libs.mediapipe.tasks.vision) } diff --git a/app/src/main/java/com/openipc/pixelpilot/ObjectDetectorHelper.java b/app/src/main/java/com/openipc/pixelpilot/ObjectDetectorHelper.java new file mode 100644 index 00000000..ca04dff2 --- /dev/null +++ b/app/src/main/java/com/openipc/pixelpilot/ObjectDetectorHelper.java @@ -0,0 +1,109 @@ +package com.openipc.pixelpilot; + +import android.content.Context; +import android.graphics.Bitmap; +import android.os.SystemClock; +import android.util.Log; + +import com.google.mediapipe.framework.image.BitmapImageBuilder; +import com.google.mediapipe.framework.image.MPImage; +import com.google.mediapipe.tasks.core.BaseOptions; +import com.google.mediapipe.tasks.core.Delegate; +import com.google.mediapipe.tasks.vision.core.RunningMode; +import com.google.mediapipe.tasks.vision.objectdetector.ObjectDetector; +import com.google.mediapipe.tasks.vision.objectdetector.ObjectDetectorResult; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.List; + +public class ObjectDetectorHelper { + private static final String TAG = "ObjectDetectorHelper"; + + public static final int DELEGATE_CPU = 0; + public static final int DELEGATE_GPU = 1; + + public static final int MODEL_EFFICIENTDETV0 = 0; + public static final int MODEL_EFFICIENTDETV2 = 1; + public static final int MODEL_CUSTOM = 2; + + private final Context context; + private final float threshold; + private final int maxResults; + private final int currentDelegate; + private final ByteBuffer modelBuffer; + + private ObjectDetector objectDetector; + + public ObjectDetectorHelper(Context context, float threshold, int maxResults, int delegate, ByteBuffer modelBuffer) { + this.context = context; + this.threshold = threshold; + this.maxResults = maxResults; + this.currentDelegate = delegate; + this.modelBuffer = modelBuffer; + setupObjectDetector(); + } + + private void setupObjectDetector() { + BaseOptions.Builder baseOptionsBuilder = BaseOptions.builder(); + if (currentDelegate == DELEGATE_GPU) { + baseOptionsBuilder.setDelegate(Delegate.GPU); + } else { + baseOptionsBuilder.setDelegate(Delegate.CPU); + } + + if (modelBuffer == null) { + Log.e(TAG, "No object detection model is available."); + return; + } + modelBuffer.rewind(); + baseOptionsBuilder.setModelAssetBuffer(modelBuffer); + + try { + ObjectDetector.ObjectDetectorOptions options = ObjectDetector.ObjectDetectorOptions.builder() + .setBaseOptions(baseOptionsBuilder.build()) + .setScoreThreshold(threshold) + .setMaxResults(maxResults) + .setRunningMode(RunningMode.IMAGE) + .build(); + objectDetector = ObjectDetector.createFromOptions(context, options); + } catch (Exception e) { + Log.e(TAG, "Failed to load model: " + e.getMessage(), e); + } + } + + public ResultBundle detectImage(Bitmap image) { + if (objectDetector == null) return null; + long startTime = SystemClock.uptimeMillis(); + MPImage mpImage = new BitmapImageBuilder(image).build(); + try { + ObjectDetectorResult detectionResult = objectDetector.detect(mpImage); + long inferenceTime = SystemClock.uptimeMillis() - startTime; + return new ResultBundle(Collections.singletonList(detectionResult), inferenceTime, image.getHeight(), image.getWidth()); + } catch (Exception e) { + Log.e(TAG, "Detection failed: " + e.getMessage(), e); + return null; + } + } + + public void clear() { + if (objectDetector != null) { + objectDetector.close(); + objectDetector = null; + } + } + + public static class ResultBundle { + public final List results; + public final long inferenceTime; + public final int inputImageHeight; + public final int inputImageWidth; + + public ResultBundle(List results, long inferenceTime, int inputImageHeight, int inputImageWidth) { + this.results = results; + this.inferenceTime = inferenceTime; + this.inputImageHeight = inputImageHeight; + this.inputImageWidth = inputImageWidth; + } + } +} diff --git a/app/src/main/java/com/openipc/pixelpilot/OverlayView.java b/app/src/main/java/com/openipc/pixelpilot/OverlayView.java new file mode 100644 index 00000000..0ced971b --- /dev/null +++ b/app/src/main/java/com/openipc/pixelpilot/OverlayView.java @@ -0,0 +1,103 @@ +package com.openipc.pixelpilot; + +import android.content.Context; +import android.graphics.Canvas; +import android.graphics.Color; +import android.graphics.Paint; +import android.graphics.Rect; +import android.graphics.RectF; +import android.util.AttributeSet; +import android.view.View; + +import com.google.mediapipe.tasks.components.containers.Detection; +import com.google.mediapipe.tasks.vision.objectdetector.ObjectDetectorResult; + +public class OverlayView extends View { + private ObjectDetectorResult results; + private final Paint boxPaint = new Paint(); + private final Paint textBackgroundPaint = new Paint(); + private final Paint textPaint = new Paint(); + private float scaleFactor = 1.0f; + private final Rect bounds = new Rect(); + private int outputWidth = 0; + private int outputHeight = 0; + + private static final int BOUNDING_RECT_TEXT_PADDING = 8; + + public OverlayView(Context context, AttributeSet attrs) { + super(context, attrs); + initPaints(); + } + + private void initPaints() { + textBackgroundPaint.setColor(Color.BLACK); + textBackgroundPaint.setStyle(Paint.Style.FILL); + textBackgroundPaint.setTextSize(40f); + + textPaint.setColor(Color.WHITE); + textPaint.setStyle(Paint.Style.FILL); + textPaint.setTextSize(40f); + + boxPaint.setColor(Color.GREEN); + boxPaint.setStrokeWidth(6f); + boxPaint.setStyle(Paint.Style.STROKE); + } + + public void clear() { + results = null; + invalidate(); + } + + @Override + protected void onDraw(Canvas canvas) { + super.onDraw(canvas); + if (results == null || results.detections() == null) return; + + for (Detection detection : results.detections()) { + RectF box = detection.boundingBox(); + + // Adjust box coordinates to view scaling + float left = box.left * scaleFactor; + float top = box.top * scaleFactor; + float right = box.right * scaleFactor; + float bottom = box.bottom * scaleFactor; + + // Draw bounding box + canvas.drawRect(left, top, right, bottom, boxPaint); + + // Category text + if (!detection.categories().isEmpty()) { + String categoryName = detection.categories().get(0).categoryName(); + float score = detection.categories().get(0).score(); + String text = String.format("%s %.2f", categoryName, score); + + // Draw background rect for text + textBackgroundPaint.getTextBounds(text, 0, text.length(), bounds); + float textWidth = bounds.width(); + float textHeight = bounds.height(); + + canvas.drawRect( + left, + top, + left + textWidth + BOUNDING_RECT_TEXT_PADDING, + top + textHeight + BOUNDING_RECT_TEXT_PADDING, + textBackgroundPaint + ); + + // Draw text + canvas.drawText(text, left, top + textHeight, textPaint); + } + } + } + + public void setResults(ObjectDetectorResult detectionResults, int inputHeight, int inputWidth) { + this.results = detectionResults; + this.outputWidth = inputWidth; + this.outputHeight = inputHeight; + + // Calculate scaling factor to scale bounding boxes to match actual display size of the View + scaleFactor = Math.min(getWidth() * 1.0f / inputWidth, getHeight() * 1.0f / inputHeight); + + invalidate(); + } +} diff --git a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java index 6c8be2b5..cacfc1ba 100644 --- a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java @@ -3,6 +3,7 @@ import android.annotation.SuppressLint; import android.app.Dialog; import android.content.BroadcastReceiver; +import android.app.ActivityManager; import android.content.Context; import android.content.Intent; import android.content.IntentFilter; @@ -68,20 +69,32 @@ import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.ByteBuffer; import java.text.SimpleDateFormat; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.Locale; import java.util.Timer; import java.util.TimerTask; +import android.graphics.Bitmap; +import android.os.SystemClock; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import android.view.TextureView; +import android.graphics.SurfaceTexture; +import android.view.Surface; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -92,6 +105,15 @@ public class VideoActivity extends AppCompatActivity implements IVideoParamsChan private static final String TAG = "pixelpilot"; private static final int PICK_KEY_REQUEST_CODE = 1; private static final int PICK_DVR_REQUEST_CODE = 2; + private static final int PICK_MODEL_REQUEST_CODE = 3; + private static final String MODEL_LITE0_FILE = "efficientdet-lite0.tflite"; + private static final String MODEL_LITE2_FILE = "efficientdet-lite2.tflite"; + private static final String MODEL_LITE0_URL = "https://storage.googleapis.com/mediapipe-models/object_detector/efficientdet_lite0/float32/1/efficientdet_lite0.tflite"; + private static final String MODEL_LITE2_URL = "https://storage.googleapis.com/mediapipe-models/object_detector/efficientdet_lite2/float32/1/efficientdet_lite2.tflite"; + private static final long MODEL_LITE0_BYTES = 13836895L; + private static final long MODEL_LITE2_BYTES = 23096891L; + private static final String PREF_OD_CUSTOM_MODEL_URI = "od_custom_model_uri"; + private static final String PREF_OD_CUSTOM_MODEL_NAME = "od_custom_model_name"; private static WifiManager wifiManager; final Handler handler = new Handler(Looper.getMainLooper()); final Runnable runnable = new Runnable() { @@ -116,6 +138,13 @@ public void run() { private ConstraintSet constraintSet; private WfbNgLink wfbLink; + private ObjectDetectorHelper objectDetectorHelper; + private ExecutorService objectDetectionExecutor; + private volatile boolean isObjectDetectionEnabled = false; + private volatile boolean isDetecting = false; + private final Object detectorLock = new Object(); + private Boolean objectDetectionRuntimeSupported = null; + private static final String PREF_DRONE_USERNAME = "drone_username"; private static final String PREF_DRONE_PASSWORD = "drone_password"; private static final String PREF_DVR_FILENAME = "dvr_filename"; @@ -349,7 +378,7 @@ private void setupVRVideoPlayers() { private void setupStandardVideoPlayer() { binding.surfaceViewRight.setVisibility(View.GONE); binding.surfaceViewLeft.setVisibility(View.GONE); - binding.mainVideo.getHolder().addCallback(videoPlayer.configure1(0)); + binding.mainVideo.setSurfaceTextureListener(videoPlayer.configureTextureView(0)); } // ---------------------------------------------------------------------------- @@ -579,6 +608,9 @@ private void showSettingsMenu(View anchor) { // Help submenu setupHelpSubMenu(popup); + // Object Detection submenu + setupObjectDetectionSubMenu(popup); + popup.show(); } @@ -1325,6 +1357,10 @@ protected void onActivityResult(int requestCode, int resultCode, @Nullable Inten startDvr(dvrUri); } } + } else if (requestCode == PICK_MODEL_REQUEST_CODE && resultCode == RESULT_OK) { + if (data != null && data.getData() != null) { + handleSelectedModelUri(data); + } } else if (requestCode == 100) { // VPN_REQUEST_CODE is 100 if (resultCode == RESULT_OK) { // VPN permission granted, start the VPN service @@ -1415,6 +1451,8 @@ public void unregisterReceivers() { protected void onPause() { super.onPause(); + stopObjectDetectionLoop(); + unregisterReceivers(); videoPlayer.stop(); @@ -1454,6 +1492,10 @@ protected void onResume() { updateUdpForwardingState(); videoPlayer.startAudio(); + SharedPreferences prefs = getSharedPreferences("general", MODE_PRIVATE); + boolean odEnabled = prefs.getBoolean("od_enabled", false); + setObjectDetectionEnabled(odEnabled); + osdManager.restoreOSDConfig(); startVpnService(); @@ -1823,4 +1865,466 @@ public void onReceivedHttpAuthRequest( } }); } + + private void setupObjectDetectionSubMenu(PopupMenu popup) { + SubMenu odMenu = popup.getMenu().addSubMenu("Object Detection"); + + SharedPreferences prefs = getSharedPreferences("general", MODE_PRIVATE); + boolean odEnabled = prefs.getBoolean("od_enabled", false); + boolean runtimeSupported = isObjectDetectionRuntimeSupported(); + boolean selectedModelAvailable = isSelectedObjectDetectionModelAvailable(); + String status = getObjectDetectionStatus(runtimeSupported, selectedModelAvailable); + + MenuItem statusItem = odMenu.add(status); + statusItem.setEnabled(false); + + MenuItem enableItem = odMenu.add("Enable"); + enableItem.setCheckable(true); + enableItem.setEnabled(runtimeSupported && selectedModelAvailable); + enableItem.setChecked(odEnabled && runtimeSupported && selectedModelAvailable); + enableItem.setOnMenuItemClickListener(item -> { + boolean newState = !item.isChecked(); + item.setChecked(newState); + setObjectDetectionEnabled(newState); + return true; + }); + + SubMenu delegateMenu = odMenu.addSubMenu("Processing Unit"); + int savedDelegate = prefs.getInt("od_delegate", ObjectDetectorHelper.DELEGATE_CPU); + + MenuItem cpuItem = delegateMenu.add("CPU"); + cpuItem.setEnabled(runtimeSupported); + cpuItem.setCheckable(true); + cpuItem.setChecked(savedDelegate == ObjectDetectorHelper.DELEGATE_CPU); + cpuItem.setOnMenuItemClickListener(item -> { + prefs.edit().putInt("od_delegate", ObjectDetectorHelper.DELEGATE_CPU).apply(); + restartObjectDetector(); + return true; + }); + + MenuItem gpuItem = delegateMenu.add("GPU"); + gpuItem.setEnabled(runtimeSupported && isGpuDelegateSupported()); + gpuItem.setCheckable(true); + gpuItem.setChecked(savedDelegate == ObjectDetectorHelper.DELEGATE_GPU); + gpuItem.setOnMenuItemClickListener(item -> { + prefs.edit().putInt("od_delegate", ObjectDetectorHelper.DELEGATE_GPU).apply(); + restartObjectDetector(); + return true; + }); + + SubMenu modelMenu = odMenu.addSubMenu("Model"); + int savedModel = prefs.getInt("od_model", ObjectDetectorHelper.MODEL_EFFICIENTDETV0); + + MenuItem downloadItem = modelMenu.add("Download Default models"); + downloadItem.setEnabled(runtimeSupported); + downloadItem.setOnMenuItemClickListener(item -> { + downloadObjectDetectionModels(); + return true; + }); + + MenuItem v0Item = modelMenu.add("EfficientDet-Lite0"); + v0Item.setEnabled(runtimeSupported && isDownloadedModelAvailable(ObjectDetectorHelper.MODEL_EFFICIENTDETV0)); + v0Item.setCheckable(true); + v0Item.setChecked(savedModel == ObjectDetectorHelper.MODEL_EFFICIENTDETV0); + v0Item.setOnMenuItemClickListener(item -> { + prefs.edit().putInt("od_model", ObjectDetectorHelper.MODEL_EFFICIENTDETV0).apply(); + restartObjectDetector(); + return true; + }); + + MenuItem v2Item = modelMenu.add("EfficientDet-Lite2"); + v2Item.setEnabled(runtimeSupported && isDownloadedModelAvailable(ObjectDetectorHelper.MODEL_EFFICIENTDETV2)); + v2Item.setCheckable(true); + v2Item.setChecked(savedModel == ObjectDetectorHelper.MODEL_EFFICIENTDETV2); + v2Item.setOnMenuItemClickListener(item -> { + prefs.edit().putInt("od_model", ObjectDetectorHelper.MODEL_EFFICIENTDETV2).apply(); + restartObjectDetector(); + return true; + }); + + MenuItem selectLocalItem = modelMenu.add("Select local model"); + selectLocalItem.setEnabled(runtimeSupported); + selectLocalItem.setOnMenuItemClickListener(item -> { + selectLocalObjectDetectionModel(); + return true; + }); + + if (isCustomModelAvailable()) { + String customModelName = prefs.getString(PREF_OD_CUSTOM_MODEL_NAME, "Local model"); + MenuItem customItem = modelMenu.add(customModelName); + customItem.setEnabled(runtimeSupported); + customItem.setCheckable(true); + customItem.setChecked(savedModel == ObjectDetectorHelper.MODEL_CUSTOM); + customItem.setOnMenuItemClickListener(item -> { + prefs.edit().putInt("od_model", ObjectDetectorHelper.MODEL_CUSTOM).apply(); + restartObjectDetector(); + return true; + }); + } + + + } + + private void setObjectDetectionEnabled(boolean enabled) { + SharedPreferences prefs = getSharedPreferences("general", MODE_PRIVATE); + if (enabled) { + boolean runtimeSupported = isObjectDetectionRuntimeSupported(); + boolean modelAvailable = isSelectedObjectDetectionModelAvailable(); + if (!runtimeSupported || !modelAvailable) { + isObjectDetectionEnabled = false; + prefs.edit().putBoolean("od_enabled", false).apply(); + binding.detectionOverlay.setVisibility(View.GONE); + binding.detectionOverlay.clear(); + stopObjectDetectionLoop(); + Toast.makeText(this, getObjectDetectionStatus(runtimeSupported, modelAvailable), Toast.LENGTH_LONG).show(); + return; + } + } + + isObjectDetectionEnabled = enabled; + prefs.edit().putBoolean("od_enabled", enabled).apply(); + + if (enabled) { + binding.detectionOverlay.setVisibility(View.VISIBLE); + startObjectDetectionLoop(); + } else { + binding.detectionOverlay.setVisibility(View.GONE); + binding.detectionOverlay.clear(); + stopObjectDetectionLoop(); + } + } + + private void restartObjectDetector() { + if (isObjectDetectionEnabled) { + stopObjectDetectionLoop(); + startObjectDetectionLoop(); + } + } + + private void startObjectDetectionLoop() { + if (isVRMode) return; // Standard mode only + if (objectDetectionExecutor == null) { + objectDetectionExecutor = Executors.newSingleThreadExecutor(); + } + SharedPreferences prefs = getSharedPreferences("general", MODE_PRIVATE); + int delegate = prefs.getInt("od_delegate", ObjectDetectorHelper.DELEGATE_CPU); + if (delegate == ObjectDetectorHelper.DELEGATE_GPU && !isGpuDelegateSupported()) { + delegate = ObjectDetectorHelper.DELEGATE_CPU; + prefs.edit().putInt("od_delegate", ObjectDetectorHelper.DELEGATE_CPU).apply(); + } + final int detectorDelegate = delegate; + + objectDetectionExecutor.execute(() -> { + ByteBuffer modelBuffer = readSelectedObjectDetectionModel(); + if (modelBuffer == null) { + runOnUiThread(() -> setObjectDetectionEnabled(false)); + return; + } + + synchronized (detectorLock) { + if (objectDetectorHelper != null) { + objectDetectorHelper.clear(); + } + objectDetectorHelper = new ObjectDetectorHelper( + VideoActivity.this, + 0.5f, + 3, + detectorDelegate, + modelBuffer + ); + } + + isDetecting = true; + while (isDetecting && isObjectDetectionEnabled) { + Bitmap bitmap = null; + try { + long start = SystemClock.uptimeMillis(); + if (binding.mainVideo != null && binding.mainVideo.isAvailable()) { + bitmap = binding.mainVideo.getBitmap(); + } + + if (bitmap != null) { + ObjectDetectorHelper.ResultBundle result = null; + synchronized (detectorLock) { + if (objectDetectorHelper != null) { + result = objectDetectorHelper.detectImage(bitmap); + } + } + + if (result != null && !result.results.isEmpty() && isObjectDetectionEnabled) { + final ObjectDetectorHelper.ResultBundle finalResult = result; + runOnUiThread(() -> { + if (isObjectDetectionEnabled) { + binding.detectionOverlay.setResults(finalResult.results.get(0), finalResult.inputImageHeight, finalResult.inputImageWidth); + } + }); + } else { + runOnUiThread(() -> { + if (isObjectDetectionEnabled) { + binding.detectionOverlay.clear(); + } + }); + } + } + + long sleepTime = 100 - (SystemClock.uptimeMillis() - start); // ~10 FPS + if (sleepTime > 0) { + Thread.sleep(sleepTime); + } + } catch (InterruptedException e) { + break; + } catch (Exception e) { + Log.e(TAG, "Error in detection loop", e); + } finally { + if (bitmap != null) { + bitmap.recycle(); + } + } + } + }); + } + + private void stopObjectDetectionLoop() { + isDetecting = false; + if (objectDetectionExecutor != null) { + objectDetectionExecutor.shutdownNow(); + objectDetectionExecutor = null; + } + synchronized (detectorLock) { + if (objectDetectorHelper != null) { + objectDetectorHelper.clear(); + objectDetectorHelper = null; + } + } + } + + private File getObjectDetectionModelsDir() { + File modelsDir = new File(getFilesDir(), "models"); + if (!modelsDir.exists() && !modelsDir.mkdirs()) { + Log.e(TAG, "Failed to create object detection models directory " + modelsDir); + } + return modelsDir; + } + + private File getDownloadedModelFile(int model) { + String fileName = model == ObjectDetectorHelper.MODEL_EFFICIENTDETV2 ? MODEL_LITE2_FILE : MODEL_LITE0_FILE; + return new File(getObjectDetectionModelsDir(), fileName); + } + + private boolean isDownloadedModelAvailable(int model) { + File file = getDownloadedModelFile(model); + long expectedBytes = model == ObjectDetectorHelper.MODEL_EFFICIENTDETV2 ? MODEL_LITE2_BYTES : MODEL_LITE0_BYTES; + return file.isFile() && file.length() == expectedBytes; + } + + private boolean isCustomModelAvailable() { + String uriString = getSharedPreferences("general", MODE_PRIVATE).getString(PREF_OD_CUSTOM_MODEL_URI, null); + if (uriString == null) { + return false; + } + try (InputStream inputStream = getContentResolver().openInputStream(Uri.parse(uriString))) { + return inputStream != null; + } catch (Exception e) { + Log.e(TAG, "Custom object detection model is not available", e); + return false; + } + } + + private boolean isSelectedObjectDetectionModelAvailable() { + SharedPreferences prefs = getSharedPreferences("general", MODE_PRIVATE); + int model = prefs.getInt("od_model", ObjectDetectorHelper.MODEL_EFFICIENTDETV0); + if (model == ObjectDetectorHelper.MODEL_CUSTOM) { + return isCustomModelAvailable(); + } + return isDownloadedModelAvailable(model); + } + + private String getObjectDetectionStatus(boolean runtimeSupported, boolean selectedModelAvailable) { + if (!runtimeSupported) { + return "Unavailable: device is not supported"; + } + if (!selectedModelAvailable) { + return "Unavailable: model not installed"; + } + return "Ready"; + } + + private boolean isObjectDetectionRuntimeSupported() { + if (objectDetectionRuntimeSupported != null) { + return objectDetectionRuntimeSupported; + } + if (!isObjectDetectionAbiSupported()) { + objectDetectionRuntimeSupported = false; + return false; + } + try { + Class.forName("com.google.mediapipe.tasks.vision.objectdetector.ObjectDetector"); + System.loadLibrary("mediapipe_tasks_vision_jni"); + objectDetectionRuntimeSupported = true; + return true; + } catch (Throwable e) { + Log.e(TAG, "Object detection runtime is not available", e); + objectDetectionRuntimeSupported = false; + return false; + } + } + + private boolean isObjectDetectionAbiSupported() { + return Arrays.asList(Build.SUPPORTED_64_BIT_ABIS).contains("arm64-v8a"); + } + + private boolean isGpuDelegateSupported() { + ActivityManager activityManager = (ActivityManager) getSystemService(Context.ACTIVITY_SERVICE); + if (activityManager == null || activityManager.getDeviceConfigurationInfo() == null) { + return false; + } + return activityManager.getDeviceConfigurationInfo().reqGlEsVersion >= 0x00030001; + } + + private void downloadObjectDetectionModels() { + Toast.makeText(this, "Downloading object detection models...", Toast.LENGTH_SHORT).show(); + new Thread(() -> { + try { + downloadObjectDetectionModel(ObjectDetectorHelper.MODEL_EFFICIENTDETV0); + downloadObjectDetectionModel(ObjectDetectorHelper.MODEL_EFFICIENTDETV2); + if (!isSelectedObjectDetectionModelAvailable()) { + getSharedPreferences("general", MODE_PRIVATE) + .edit() + .putInt("od_model", ObjectDetectorHelper.MODEL_EFFICIENTDETV0) + .apply(); + } + runOnUiThread(() -> Toast.makeText(this, "Object detection models downloaded", Toast.LENGTH_LONG).show()); + } catch (IOException e) { + Log.e(TAG, "Failed to download object detection models", e); + runOnUiThread(() -> Toast.makeText(this, "Failed to download models: " + e.getMessage(), Toast.LENGTH_LONG).show()); + } + }).start(); + } + + private void downloadObjectDetectionModel(int model) throws IOException { + File target = getDownloadedModelFile(model); + long expectedBytes = model == ObjectDetectorHelper.MODEL_EFFICIENTDETV2 ? MODEL_LITE2_BYTES : MODEL_LITE0_BYTES; + if (target.isFile() && target.length() == expectedBytes) { + return; + } + + String modelUrl = model == ObjectDetectorHelper.MODEL_EFFICIENTDETV2 ? MODEL_LITE2_URL : MODEL_LITE0_URL; + File tempFile = new File(target.getParentFile(), target.getName() + ".download"); + HttpURLConnection connection = (HttpURLConnection) new URL(modelUrl).openConnection(); + connection.setConnectTimeout(15000); + connection.setReadTimeout(60000); + connection.connect(); + int responseCode = connection.getResponseCode(); + if (responseCode < 200 || responseCode >= 300) { + throw new IOException("HTTP " + responseCode); + } + + long bytesCopied = 0; + try (InputStream inputStream = connection.getInputStream(); + OutputStream outputStream = new FileOutputStream(tempFile)) { + byte[] buffer = new byte[64 * 1024]; + int read; + while ((read = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, read); + bytesCopied += read; + } + } finally { + connection.disconnect(); + } + + if (bytesCopied != expectedBytes) { + if (!tempFile.delete()) { + Log.w(TAG, "Failed to delete incomplete model " + tempFile); + } + throw new IOException("Incomplete model download"); + } + if (target.exists() && !target.delete()) { + throw new IOException("Failed to replace existing model"); + } + if (!tempFile.renameTo(target)) { + throw new IOException("Failed to save model"); + } + } + + private void selectLocalObjectDetectionModel() { + Intent intent = new Intent(Intent.ACTION_OPEN_DOCUMENT); + intent.addCategory(Intent.CATEGORY_OPENABLE); + intent.setType("*/*"); + intent.putExtra(Intent.EXTRA_MIME_TYPES, new String[]{"application/octet-stream", "application/x-tflite"}); + intent.addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION | Intent.FLAG_GRANT_PERSISTABLE_URI_PERMISSION); + startActivityForResult(intent, PICK_MODEL_REQUEST_CODE); + } + + private void handleSelectedModelUri(Intent data) { + Uri uri = data.getData(); + if (uri == null) { + return; + } + String displayName = "Local model"; + DocumentFile documentFile = DocumentFile.fromSingleUri(this, uri); + if (documentFile != null && documentFile.getName() != null) { + displayName = documentFile.getName(); + } + if (!displayName.toLowerCase(Locale.US).endsWith(".tflite")) { + Toast.makeText(this, "Please select a .tflite model file", Toast.LENGTH_LONG).show(); + return; + } + + try (InputStream inputStream = getContentResolver().openInputStream(uri)) { + if (inputStream == null) { + throw new IOException("Unable to open model file"); + } + final int takeFlags = data.getFlags() & Intent.FLAG_GRANT_READ_URI_PERMISSION; + getContentResolver().takePersistableUriPermission(uri, takeFlags); + SharedPreferences.Editor editor = getSharedPreferences("general", MODE_PRIVATE).edit(); + editor.putString(PREF_OD_CUSTOM_MODEL_URI, uri.toString()); + editor.putString(PREF_OD_CUSTOM_MODEL_NAME, displayName); + editor.putInt("od_model", ObjectDetectorHelper.MODEL_CUSTOM); + editor.apply(); + Toast.makeText(this, "Selected " + displayName, Toast.LENGTH_LONG).show(); + restartObjectDetector(); + } catch (Exception e) { + Log.e(TAG, "Failed to select local object detection model", e); + Toast.makeText(this, "Failed to select model: " + e.getMessage(), Toast.LENGTH_LONG).show(); + } + } + + private ByteBuffer readSelectedObjectDetectionModel() { + SharedPreferences prefs = getSharedPreferences("general", MODE_PRIVATE); + int model = prefs.getInt("od_model", ObjectDetectorHelper.MODEL_EFFICIENTDETV0); + try { + if (model == ObjectDetectorHelper.MODEL_CUSTOM) { + String uriString = prefs.getString(PREF_OD_CUSTOM_MODEL_URI, null); + if (uriString == null) { + return null; + } + try (InputStream inputStream = getContentResolver().openInputStream(Uri.parse(uriString))) { + return readModelBuffer(inputStream); + } + } + try (InputStream inputStream = new FileInputStream(getDownloadedModelFile(model))) { + return readModelBuffer(inputStream); + } + } catch (IOException e) { + Log.e(TAG, "Failed to read object detection model", e); + return null; + } + } + + private ByteBuffer readModelBuffer(InputStream inputStream) throws IOException { + if (inputStream == null) { + throw new IOException("Model input stream is null"); + } + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + byte[] buffer = new byte[64 * 1024]; + int read; + while ((read = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, read); + } + byte[] bytes = outputStream.toByteArray(); + ByteBuffer directBuffer = ByteBuffer.allocateDirect(bytes.length); + directBuffer.put(bytes); + directBuffer.rewind(); + return directBuffer; + } } diff --git a/app/src/main/res/layout/activity_video.xml b/app/src/main/res/layout/activity_video.xml index f888475f..cabe29e6 100644 --- a/app/src/main/res/layout/activity_video.xml +++ b/app/src/main/res/layout/activity_video.xml @@ -5,7 +5,7 @@ android:layout_width="match_parent" android:layout_height="match_parent"> - + +