Add AI Agent Android app: accessibility automation, planner, Gemini API client and UI

app/build.gradle.kts

@@ -0,0 +1,51 @@
+plugins {
+    id("com.android.application")
+    id("org.jetbrains.kotlin.android")
+}
+
+android {
+    namespace = "com.aquib.aiagent"
+    compileSdk = 35
+
+    defaultConfig {
+        applicationId = "com.aquib.aiagent"
+        minSdk = 31
+        targetSdk = 35
+        versionCode = 1
+        versionName = "1.0"
+
+        testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
+    }
+
+    buildTypes {
+        release {
+            isMinifyEnabled = false
+            proguardFiles(
+                getDefaultProguardFile("proguard-android-optimize.txt"),
+                "proguard-rules.pro"
+            )
+        }
+    }
+
+    compileOptions {
+        sourceCompatibility = JavaVersion.VERSION_17
+        targetCompatibility = JavaVersion.VERSION_17
+    }
+    kotlinOptions {
+        jvmTarget = "17"
+    }
+    buildFeatures {
+        viewBinding = true
+    }
+}
+
+dependencies {
+    implementation("androidx.core:core-ktx:1.13.1")
+    implementation("androidx.appcompat:appcompat:1.7.0")
+    implementation("androidx.activity:activity-ktx:1.9.2")
+    implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.8.4")
+    implementation("com.google.android.material:material:1.12.0")
+    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.8.1")
+    implementation("com.google.ai.client.generativeai:generativeai:0.9.0")
+    implementation("androidx.security:security-crypto:1.1.0-alpha06")
+}

app/proguard-rules.pro

@@ -0,0 +1 @@
+# Project specific rules

app/src/main/AndroidManifest.xml

@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools">
+
+    <uses-permission android:name="android.permission.SYSTEM_ALERT_WINDOW" />
+    <uses-permission android:name="android.permission.INTERNET" />
+    <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
+    <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION" />
+    <uses-permission android:name="android.permission.RECORD_AUDIO" />
+    <uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
+    <uses-permission android:name="android.permission.WAKE_LOCK" />
+    <uses-permission android:name="android.permission.REQUEST_IGNORE_BATTERY_OPTIMIZATIONS" />
+
+    <application
+        android:name=".AiAgentApp"
+        android:allowBackup="true"
+        android:icon="@mipmap/ic_launcher"
+        android:label="@string/app_name"
+        android:roundIcon="@mipmap/ic_launcher_round"
+        android:supportsRtl="true"
+        android:theme="@style/Theme.AiAgent"
+        tools:targetApi="31">
+
+        <activity
+            android:name=".MainActivity"
+            android:exported="true">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+
+        <service
+            android:name=".accessibility.AccessibilityAgentService"
+            android:exported="false"
+            android:permission="android.permission.BIND_ACCESSIBILITY_SERVICE">
+            <intent-filter>
+                <action android:name="android.accessibilityservice.AccessibilityService" />
+            </intent-filter>
+            <meta-data
+                android:name="android.accessibilityservice"
+                android:resource="@xml/accessibility_service_config" />
+        </service>
+
+        <service
+            android:name=".overlay.AgentOverlayService"
+            android:enabled="true"
+            android:exported="false"
+            android:foregroundServiceType="mediaProjection" />
+    </application>
+
+</manifest>

app/src/main/java/com/aquib/aiagent/AiAgentApp.kt

@@ -0,0 +1,11 @@
+package com.aquib.aiagent
+
+import android.app.Application
+import com.aquib.aiagent.util.NotificationHelper
+
+class AiAgentApp : Application() {
+    override fun onCreate() {
+        super.onCreate()
+        NotificationHelper.createChannel(this)
+    }
+}

app/src/main/java/com/aquib/aiagent/MainActivity.kt

@@ -0,0 +1,126 @@
+package com.aquib.aiagent
+
+import android.content.Intent
+import android.content.pm.PackageManager
+import android.net.Uri
+import android.os.Build
+import android.os.Bundle
+import android.os.PowerManager
+import android.provider.Settings
+import android.widget.Button
+import android.widget.EditText
+import android.widget.TextView
+import android.widget.Toast
+import androidx.activity.ComponentActivity
+import androidx.core.content.ContextCompat
+import androidx.lifecycle.lifecycleScope
+import com.aquib.aiagent.ai.GeminiApiClient
+import com.aquib.aiagent.overlay.AgentOverlayService
+import com.aquib.aiagent.telemetry.Telemetry
+import com.aquib.aiagent.util.SecurePrefs
+import kotlinx.coroutines.launch
+
+class MainActivity : ComponentActivity() {
+    private lateinit var securePrefs: SecurePrefs
+    private lateinit var api: GeminiApiClient
+    private lateinit var telemetry: Telemetry
+
+    override fun onCreate(savedInstanceState: Bundle?) {
+        super.onCreate(savedInstanceState)
+        setContentView(R.layout.activity_main)
+
+        securePrefs = SecurePrefs(this)
+        api = GeminiApiClient(this)
+        telemetry = Telemetry(this)
+
+        val tvChecklist = findViewById<TextView>(R.id.tvChecklist)
+        val tvStats = findViewById<TextView>(R.id.tvStats)
+        val etApiKey = findViewById<EditText>(R.id.etApiKey)
+        val btnSave = findViewById<Button>(R.id.btnSaveKey)
+        val btnFixBattery = findViewById<Button>(R.id.btnFixBattery)
+
+        etApiKey.setText(securePrefs.getString("active_api_key"))
+        btnSave.setOnClickListener {
+            securePrefs.putString("active_api_key", etApiKey.text?.toString().orEmpty().trim())
+            refreshStats(tvStats)
+            Toast.makeText(this, "API key saved", Toast.LENGTH_SHORT).show()
+        }
+
+        btnFixBattery.setOnClickListener { requestBatteryExemption() }
+
+        requestRuntimePermissions()
+        maybeStartOverlayService()
+        tvChecklist.text = buildChecklist()
+        refreshStats(tvStats)
+    }
+
+    override fun onResume() {
+        super.onResume()
+        findViewById<TextView>(R.id.tvChecklist).text = buildChecklist()
+        refreshStats(findViewById(R.id.tvStats))
+    }
+
+    private fun requestRuntimePermissions() {
+        val needed = mutableListOf<String>()
+        if (checkSelfPermission(android.Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) {
+            needed += android.Manifest.permission.RECORD_AUDIO
+        }
+        if (Build.VERSION.SDK_INT >= 33 && checkSelfPermission(android.Manifest.permission.POST_NOTIFICATIONS) != PackageManager.PERMISSION_GRANTED) {
+            needed += android.Manifest.permission.POST_NOTIFICATIONS
+        }
+        if (needed.isNotEmpty()) requestPermissions(needed.toTypedArray(), 7)
+    }
+
+    private fun refreshStats(tvStats: TextView) {
+        lifecycleScope.launch {
+            val lite = api.dailyStats(complex = false)
+            val remaining = lite.limit - lite.used
+            val pct = if (lite.limit == 0) 0f else (remaining.toFloat() / lite.limit)
+            val color = when {
+                pct < 0.1f -> R.color.danger
+                pct < 0.2f -> R.color.warn
+                else -> R.color.text_primary
+            }
+            tvStats.setTextColor(ContextCompat.getColor(this@MainActivity, color))
+            tvStats.text = buildString {
+                appendLine("API calls today: ${lite.used} / ${lite.limit} remaining: $remaining")
+                appendLine(telemetry.summary())
+            }
+        }
+    }
+
+    private fun buildChecklist(): String {
+        val powerManager = getSystemService(PowerManager::class.java)
+        val batteryOk = powerManager.isIgnoringBatteryOptimizations(packageName)
+        val overlayOk = Settings.canDrawOverlays(this)
+        val accessibilityOk = Settings.Secure.getString(contentResolver, Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES)
+            ?.contains(packageName) == true
+        val micOk = checkSelfPermission(android.Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED
+
+        val notifOk = if (Build.VERSION.SDK_INT >= 33) {
+            checkSelfPermission(android.Manifest.permission.POST_NOTIFICATIONS) == PackageManager.PERMISSION_GRANTED
+        } else true
+
+        return """
+Overlay: ${status(overlayOk)}
+Accessibility: ${status(accessibilityOk)}
+Microphone: ${status(micOk)}
+Notifications: ${status(notifOk)}
+Battery Optimization Exemption: ${status(batteryOk)}
+""".trimIndent()
+    }
+
+    private fun status(value: Boolean): String = if (value) "✅ Enabled" else "❌ Missing"
+
+    private fun requestBatteryExemption() {
+        val intent = Intent(Settings.ACTION_REQUEST_IGNORE_BATTERY_OPTIMIZATIONS)
+            .setData(Uri.parse("package:$packageName"))
+        startActivity(intent)
+    }
+
+    private fun maybeStartOverlayService() {
+        if (!Settings.canDrawOverlays(this)) return
+        val overlayIntent = Intent(this, AgentOverlayService::class.java)
+        ContextCompat.startForegroundService(this, overlayIntent)
+    }
+}

app/src/main/java/com/aquib/aiagent/accessibility/AccessibilityAgentService.kt

@@ -0,0 +1,72 @@
+package com.aquib.aiagent.accessibility
+
+import android.accessibilityservice.AccessibilityService
+import android.accessibilityservice.GestureDescription
+import android.graphics.Path
+import android.os.Bundle
+import android.view.accessibility.AccessibilityEvent
+import android.view.accessibility.AccessibilityNodeInfo
+
+class AccessibilityAgentService : AccessibilityService() {
+
+    companion object {
+        @Volatile
+        var instance: AccessibilityAgentService? = null
+            private set
+    }
+
+    override fun onServiceConnected() {
+        super.onServiceConnected()
+        instance = this
+    }
+
+    override fun onAccessibilityEvent(event: AccessibilityEvent?) = Unit
+
+    override fun onInterrupt() = Unit
+
+    override fun onDestroy() {
+        super.onDestroy()
+        if (instance === this) instance = null
+    }
+
+    fun root(): AccessibilityNodeInfo? = rootInActiveWindow
+
+    fun performTap(x: Int, y: Int): Boolean {
+        val path = Path().apply { moveTo(x.toFloat(), y.toFloat()) }
+        val gesture = GestureDescription.Builder()
+            .addStroke(GestureDescription.StrokeDescription(path, 0, 60))
+            .build()
+        return dispatchGesture(gesture, null, null)
+    }
+
+    fun performSwipe(startX: Int, startY: Int, endX: Int, endY: Int, durationMs: Long = 300): Boolean {
+        val path = Path().apply {
+            moveTo(startX.toFloat(), startY.toFloat())
+            lineTo(endX.toFloat(), endY.toFloat())
+        }
+        val gesture = GestureDescription.Builder()
+            .addStroke(GestureDescription.StrokeDescription(path, 0, durationMs))
+            .build()
+        return dispatchGesture(gesture, null, null)
+    }
+
+    fun performType(text: String): Boolean {
+        val node = findInputNode(rootInActiveWindow) ?: return false
+        val args = Bundle().apply { putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text) }
+        return node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, args)
+    }
+
+    fun performBack(): Boolean = performGlobalAction(GLOBAL_ACTION_BACK)
+
+    fun performHome(): Boolean = performGlobalAction(GLOBAL_ACTION_HOME)
+
+    private fun findInputNode(node: AccessibilityNodeInfo?): AccessibilityNodeInfo? {
+        node ?: return null
+        if (node.isEditable) return node
+        for (i in 0 until node.childCount) {
+            val found = findInputNode(node.getChild(i))
+            if (found != null) return found
+        }
+        return null
+    }
+}

app/src/main/java/com/aquib/aiagent/agent/ActionVerifier.kt

@@ -0,0 +1,30 @@
+package com.aquib.aiagent.agent
+
+import android.graphics.Bitmap
+import android.graphics.BitmapFactory
+import kotlin.math.abs
+
+class ActionVerifier {
+    fun changed(beforeBytes: ByteArray, afterBytes: ByteArray): Boolean {
+        var before: Bitmap? = null
+        var after: Bitmap? = null
+        return try {
+            before = BitmapFactory.decodeByteArray(beforeBytes, 0, beforeBytes.size)
+            after = BitmapFactory.decodeByteArray(afterBytes, 0, afterBytes.size)
+            if (before == null || after == null) return true
+            if (before.width != after.width || before.height != after.height) return true
+
+            val samplePoints = listOf(0.2f to 0.2f, 0.5f to 0.5f, 0.8f to 0.8f, 0.3f to 0.7f, 0.7f to 0.3f)
+            var delta = 0
+            samplePoints.forEach { (xf, yf) ->
+                val x = (before.width * xf).toInt().coerceIn(0, before.width - 1)
+                val y = (before.height * yf).toInt().coerceIn(0, before.height - 1)
+                delta += abs(before.getPixel(x, y) - after.getPixel(x, y))
+            }
+            delta > 100
+        } finally {
+            before?.recycle()
+            after?.recycle()
+        }
+    }
+}

app/src/main/java/com/aquib/aiagent/agent/AdaptiveWaiter.kt

@@ -0,0 +1,30 @@
+package com.aquib.aiagent.agent
+
+import android.os.SystemClock
+import kotlinx.coroutines.delay
+
+class AdaptiveWaiter {
+    suspend fun waitUntil(timeoutMs: Long = 10_000L, pollMs: Long = 250L, condition: () -> Boolean): Boolean {
+        val start = SystemClock.elapsedRealtime()
+        while (SystemClock.elapsedRealtime() - start < timeoutMs) {
+            if (condition()) return true
+            delay(pollMs)
+        }
+        return false
+    }
+
+    suspend fun waitForSettle(snapshot: () -> Int, timeoutMs: Long = 8_000L): Boolean {
+        var stableCount = 0
+        var last = snapshot()
+        return waitUntil(timeoutMs = timeoutMs, pollMs = 300L) {
+            val current = snapshot()
+            if (current == last) {
+                stableCount++
+            } else {
+                stableCount = 0
+                last = current
+            }
+            stableCount >= 3
+        }
+    }
+}