diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..aa724b7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +*.iml +.gradle +/local.properties +/.idea/caches +/.idea/libraries +/.idea/modules.xml +/.idea/workspace.xml +/.idea/navEditor.xml +/.idea/assetWizardSettings.xml +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..26d3352 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/.idea/.name b/.idea/.name new file mode 100644 index 0000000..3303b11 --- /dev/null +++ b/.idea/.name @@ -0,0 +1 @@ +smart blind stick \ No newline at end of file diff --git a/.idea/AndroidProjectSystem.xml b/.idea/AndroidProjectSystem.xml new file mode 100644 index 0000000..4a53bee --- /dev/null +++ b/.idea/AndroidProjectSystem.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml new file mode 100644 index 0000000..7643783 --- /dev/null +++ b/.idea/codeStyles/Project.xml @@ -0,0 +1,123 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000..79ee123 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/.idea/compiler.xml b/.idea/compiler.xml new file mode 100644 index 0000000..b589d56 --- /dev/null +++ b/.idea/compiler.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/deploymentTargetSelector.xml b/.idea/deploymentTargetSelector.xml new file mode 100644 index 0000000..ca16a99 --- /dev/null +++ b/.idea/deploymentTargetSelector.xml @@ -0,0 +1,11 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/gradle.xml b/.idea/gradle.xml new file mode 100644 index 0000000..02c4aa5 --- /dev/null +++ b/.idea/gradle.xml @@ -0,0 +1,18 @@ + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..f0c6ad0 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,50 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..31e5cb0 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,9 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/runConfigurations.xml b/.idea/runConfigurations.xml new file mode 100644 index 0000000..16660f1 --- /dev/null +++ b/.idea/runConfigurations.xml @@ -0,0 +1,17 @@ + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/README.md b/README.md index ac214cd..f0f687f 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,10 @@ -# 🚀 TechQuest — Team Submission - -> **Fork this repo → build your app on your fork → open a Pull Request back to `Reskilll/TechQuest`.** -> Your Pull Request is your official submission. Fill in this README as you go. - ---- - ## 👥 Team | | | |---|---| -| **Team name** | _e.g. Team Nova_ | -| **Members** | _Name 1, Name 2, Name 3_ | -| **City / Venue** | _Pune / Hyderabad / Bengaluru / Chennai_ | +| **Team Name** | Clover Dynamics | +| **Team Member** | Kamakhya Anupam Sharma | +| **City** | Bengaluru | --- @@ -19,35 +12,35 @@ | | | |---|---| -| **App name** | _Your app name_ | -| **Theme** | _Pick one:_ Simple game · Utility app · Productivity · Fun & social · Learning tool · Creative tool | -| **One-liner** | _What your app does, in one sentence._ | +| **App name** | Smart Blind Stick Android App | +| **Theme** | Utility app | +| **One-liner** | It's an assistive technology app that helps visually impaired users sense their environment through the camera of their phones | ### What we built -_A short paragraph: what the app does and who it's for._ +A comprehensive assistive Android application designed to give visually impaired users independence. The app uses offline on-device computer vision (YOLOv8) combined with a custom 3D spatial audio engine to give users a real-time "acoustic map" of their surroundings. It features a Voice Assistant that leverages powerful Cloud Vision AI to describe environments, read text (OCR), and provide micro-navigation (e.g., finding a door). Additionally, the app includes an automatic Fall Detection system that sends a Telegram SOS message with the user's GPS location, a persistent Spatial Memory database, and an offline Facial Recognition engine to identify friends. ### How the AI is used -- **Model:** _e.g. `openai/gpt-4o-mini` (via OpenRouter)_ -- **What the AI does:** _e.g. generates quiz questions from a topic the user types._ -- **AI pattern:** _Chat · Summarise · Classify · Generate · Extract · Vision_ +- **Model:** `meta/llama-3.2-90b-vision-instruct` (via Nvidia NIM API) +- **What the AI does:** Acts as the cognitive brain of the app. It takes an image of the user's surroundings along with a voice prompt (e.g., "Describe what you see" or "Find a door") and provides a detailed 3x3 grid spatial description to the user via Text-To-Speech. +- **AI pattern:** Vision · Chat · Extract --- ## ▶️ How to run it ```bash -# 1. Clone (your fork) -git clone https://github.com//TechQuest.git +# 1. Clone the repository +git clone https://github.com/k-anupam-sharma/TechQuest.git cd TechQuest -# 2. Add your OpenRouter key as described below, then open in Android Studio +# 2. Open the project in Android Studio # (let Gradle finish syncing before you Run) ``` -**OpenRouter setup** -- Base URL: `https://openrouter.ai/api/v1` -- Model: `_your model_` -- API key: stored as _e.g. a `local.properties` value / environment variable_ — **never committed** (see below). +**API Setup** +- The Cloud Vision features require an API key. +- Open `app/src/main/java/com/example/smartblindstick/NvidiaApiClient.kt` +- Replace the `apiKey` variable with your Nvidia NIM API key. **Build the APK** - Android Studio → `Build → Build Bundle(s)/APK(s) → Build APK(s)` @@ -57,26 +50,30 @@ cd TechQuest ## 📱 Demo -- **APK:** _link or `app-debug.apk` in the repo_ -- **Screen recording:** _link to a short video of the app running on the iQOO_ -- **Screenshots:** _optional_ +- **APK:** Navigate to the `app/build/outputs/apk/debug/app-debug.apk` path after building to install the app on your device. +- **Screen recording:** *(Add your video link here)* +- **Screenshots:** *(Add your screenshots here)* --- ## 🧰 Tech stack -- Android (Kotlin / Java) · Android Studio -- OpenRouter API (LLM) -- _anything else you used_ +- Android (Kotlin) · Android Studio +- Nvidia NIM API (Meta LLaMA Vision) +- TensorFlow Lite (YOLOv8 Object Detection, MobileFaceNet) +- Google ML Kit (Face Detection & Text Recognition OCR) +- Android CameraX & SensorManager +- Room Database (SQLite) +- OkHttp (Networking & Telegram API) --- ## ✅ Submission checklist -- [ ] This README is filled in (team, theme, how to run) -- [ ] The API key is **NOT** in the repo (see `.gitignore` below) -- [ ] Final code pushed to **your fork** +- [x] This README is filled in (team, theme, how to run) +- [x] The API key is **NOT** in the repo (see `.gitignore` below) +- [x] Final code pushed to **your fork** - [ ] APK and/or a screen recording added or linked - [ ] **Pull Request opened** from your fork → `Reskilll/TechQuest` **before the deadline** -- [ ] PR title = your **team name** +- [x] PR title = your **team name** --- diff --git a/app/.gitignore b/app/.gitignore new file mode 100644 index 0000000..42afabf --- /dev/null +++ b/app/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/app/build.gradle.kts b/app/build.gradle.kts new file mode 100644 index 0000000..0648525 --- /dev/null +++ b/app/build.gradle.kts @@ -0,0 +1,89 @@ +plugins { + alias(libs.plugins.android.application) + alias(libs.plugins.kotlin.android) + alias(libs.plugins.kotlin.compose) + alias(libs.plugins.ksp) +} + +android { + namespace = "com.example.smartblindstick" + compileSdk = 35 + + defaultConfig { + applicationId = "com.example.smartblindstick" + minSdk = 24 + targetSdk = 35 + versionCode = 1 + versionName = "1.0" + + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + } + + buildTypes { + release { + isMinifyEnabled = false + proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro") + } + } + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } + kotlinOptions { + jvmTarget = "17" + } + buildFeatures { + compose = true + } +} + +dependencies { + implementation(platform(libs.androidx.compose.bom)) + implementation(libs.androidx.activity.compose) + implementation(libs.androidx.compose.material3) + implementation(libs.androidx.compose.ui) + implementation(libs.androidx.compose.ui.graphics) + implementation(libs.androidx.compose.ui.tooling.preview) + implementation(libs.androidx.core.ktx) + implementation(libs.androidx.lifecycle.runtime.ktx) + testImplementation(libs.junit) + androidTestImplementation(platform(libs.androidx.compose.bom)) + androidTestImplementation(libs.androidx.compose.ui.test.junit4) + androidTestImplementation(libs.androidx.espresso.core) + androidTestImplementation(libs.androidx.junit) + debugImplementation(libs.androidx.compose.ui.test.manifest) + debugImplementation(libs.androidx.compose.ui.tooling) + + // CameraX core library + val cameraxVersion = "1.3.0-alpha04" + implementation("androidx.camera:camera-core:$cameraxVersion") + implementation("androidx.camera:camera-camera2:$cameraxVersion") + implementation("androidx.camera:camera-lifecycle:$cameraxVersion") + implementation("androidx.camera:camera-view:$cameraxVersion") + + // ML Kit Face Detection + implementation("com.google.mlkit:face-detection:16.1.5") + + // ML Kit Object Detection + implementation("com.google.mlkit:object-detection:17.0.0") + implementation("com.google.mlkit:object-detection-custom:17.0.0") + + // TensorFlow Lite + implementation("org.tensorflow:tensorflow-lite:2.16.1") + + // Coroutines + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.6.4") + // ML Kit Text Recognition (OCR) + implementation("com.google.mlkit:text-recognition:16.0.1") + + // OkHttp for Telegram API requests + implementation("com.squareup.okhttp3:okhttp:4.12.0") + + // Gson for simple JSON parsing (useful for face embeddings if needed) + implementation("com.google.code.gson:gson:2.10.1") + + // Room components + implementation(libs.androidx.room.runtime) + ksp(libs.androidx.room.compiler) + implementation(libs.androidx.room.ktx) +} \ No newline at end of file diff --git a/app/src/androidTest/java/com/example/smartblindstick/ExampleInstrumentedTest.kt b/app/src/androidTest/java/com/example/smartblindstick/ExampleInstrumentedTest.kt new file mode 100644 index 0000000..80733f9 --- /dev/null +++ b/app/src/androidTest/java/com/example/smartblindstick/ExampleInstrumentedTest.kt @@ -0,0 +1,24 @@ +package com.example.smartblindstick + +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.ext.junit.runners.AndroidJUnit4 + +import org.junit.Test +import org.junit.runner.RunWith + +import org.junit.Assert.* + +/** + * Instrumented test, which will execute on an Android device. + * + * See [testing documentation](http://d.android.com/tools/testing). + */ +@RunWith(AndroidJUnit4::class) +class ExampleInstrumentedTest { + @Test + fun useAppContext() { + // Context of the app under test. + val appContext = InstrumentationRegistry.getInstrumentation().targetContext + assertEquals("com.example.smartblindstick", appContext.packageName) + } +} \ No newline at end of file diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml new file mode 100644 index 0000000..e8ade78 --- /dev/null +++ b/app/src/main/AndroidManifest.xml @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/app/src/main/assets/coco_labels.txt b/app/src/main/assets/coco_labels.txt new file mode 100644 index 0000000..941cb4e --- /dev/null +++ b/app/src/main/assets/coco_labels.txt @@ -0,0 +1,80 @@ +person +bicycle +car +motorcycle +airplane +bus +train +truck +boat +traffic light +fire hydrant +stop sign +parking meter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +backpack +umbrella +handbag +tie +suitcase +frisbee +skis +snowboard +sports ball +kite +baseball bat +baseball glove +skateboard +surfboard +tennis racket +bottle +wine glass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hot dog +pizza +donut +cake +chair +couch +potted plant +bed +dining table +toilet +tv +laptop +mouse +remote +keyboard +cell phone +microwave +oven +toaster +sink +refrigerator +book +clock +vase +scissors +teddy bear +hair drier +toothbrush diff --git a/app/src/main/assets/known_faces/Adithya.jpg b/app/src/main/assets/known_faces/Adithya.jpg new file mode 100644 index 0000000..8b36d23 Binary files /dev/null and b/app/src/main/assets/known_faces/Adithya.jpg differ diff --git a/app/src/main/assets/known_faces/Anupam.jpg b/app/src/main/assets/known_faces/Anupam.jpg new file mode 100644 index 0000000..9cdd72b Binary files /dev/null and b/app/src/main/assets/known_faces/Anupam.jpg differ diff --git a/app/src/main/assets/known_faces/Aryan.jpeg b/app/src/main/assets/known_faces/Aryan.jpeg new file mode 100644 index 0000000..8b579ce Binary files /dev/null and b/app/src/main/assets/known_faces/Aryan.jpeg differ diff --git a/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.34 (1).jpeg b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.34 (1).jpeg new file mode 100644 index 0000000..57e680d Binary files /dev/null and b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.34 (1).jpeg differ diff --git a/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.34.jpeg b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.34.jpeg new file mode 100644 index 0000000..5e73fa9 Binary files /dev/null and b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.34.jpeg differ diff --git a/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.35 (1).jpeg b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.35 (1).jpeg new file mode 100644 index 0000000..3af3a27 Binary files /dev/null and b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.35 (1).jpeg differ diff --git a/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.35.jpeg b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.35.jpeg new file mode 100644 index 0000000..b71b9f7 Binary files /dev/null and b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.35.jpeg differ diff --git a/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.37.jpeg b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.37.jpeg new file mode 100644 index 0000000..5691c7d Binary files /dev/null and b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.37.jpeg differ diff --git a/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.38.jpeg b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.38.jpeg new file mode 100644 index 0000000..cb2e472 Binary files /dev/null and b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.38.jpeg differ diff --git a/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.39.jpeg b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.39.jpeg new file mode 100644 index 0000000..8453b34 Binary files /dev/null and b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.39.jpeg differ diff --git a/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.40.jpeg b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.40.jpeg new file mode 100644 index 0000000..8e6c76a Binary files /dev/null and b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.40.jpeg differ diff --git a/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.41.jpeg b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.41.jpeg new file mode 100644 index 0000000..a351d58 Binary files /dev/null and b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.41.jpeg differ diff --git a/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.42.jpeg b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.42.jpeg new file mode 100644 index 0000000..f1654af Binary files /dev/null and b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.42.jpeg differ diff --git a/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.44.jpeg b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.44.jpeg new file mode 100644 index 0000000..d53c84d Binary files /dev/null and b/app/src/main/assets/known_faces/Blind Stick Face Recog/WhatsApp Image 2026-05-24 at 14.53.44.jpeg differ diff --git a/app/src/main/assets/known_faces/Harsha.jpg b/app/src/main/assets/known_faces/Harsha.jpg new file mode 100644 index 0000000..4d07673 Binary files /dev/null and b/app/src/main/assets/known_faces/Harsha.jpg differ diff --git a/app/src/main/assets/known_faces/Kartikey.jpg b/app/src/main/assets/known_faces/Kartikey.jpg new file mode 100644 index 0000000..6af51e6 Binary files /dev/null and b/app/src/main/assets/known_faces/Kartikey.jpg differ diff --git a/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.20 (1).jpeg b/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.20 (1).jpeg new file mode 100644 index 0000000..9652674 Binary files /dev/null and b/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.20 (1).jpeg differ diff --git a/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.20.jpeg b/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.20.jpeg new file mode 100644 index 0000000..0df7813 Binary files /dev/null and b/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.20.jpeg differ diff --git a/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.21 (1).jpeg b/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.21 (1).jpeg new file mode 100644 index 0000000..7d4d0fc Binary files /dev/null and b/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.21 (1).jpeg differ diff --git a/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.21.jpeg b/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.21.jpeg new file mode 100644 index 0000000..cc4ea03 Binary files /dev/null and b/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.21.jpeg differ diff --git a/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.22 (1).jpeg b/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.22 (1).jpeg new file mode 100644 index 0000000..7bd5970 Binary files /dev/null and b/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.22 (1).jpeg differ diff --git a/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.22.jpeg b/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.22.jpeg new file mode 100644 index 0000000..433244e Binary files /dev/null and b/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.22.jpeg differ diff --git a/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.23.jpeg b/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.23.jpeg new file mode 100644 index 0000000..1ad6303 Binary files /dev/null and b/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.23.jpeg differ diff --git a/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.24 (1).jpeg b/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.24 (1).jpeg new file mode 100644 index 0000000..d605eef Binary files /dev/null and b/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.24 (1).jpeg differ diff --git a/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.24.jpeg b/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.24.jpeg new file mode 100644 index 0000000..28c2158 Binary files /dev/null and b/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.24.jpeg differ diff --git a/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.25.jpeg b/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.25.jpeg new file mode 100644 index 0000000..48d7a49 Binary files /dev/null and b/app/src/main/assets/known_faces/Nandeeshwar Sir/WhatsApp Image 2026-05-24 at 16.31.25.jpeg differ diff --git a/app/src/main/assets/mobilefacenet.tflite b/app/src/main/assets/mobilefacenet.tflite new file mode 100644 index 0000000..dcba8d1 Binary files /dev/null and b/app/src/main/assets/mobilefacenet.tflite differ diff --git a/app/src/main/assets/yolov8n.tflite b/app/src/main/assets/yolov8n.tflite new file mode 100644 index 0000000..2af1309 Binary files /dev/null and b/app/src/main/assets/yolov8n.tflite differ diff --git a/app/src/main/java/com/example/smartblindstick/AutoFlashlightManager.kt b/app/src/main/java/com/example/smartblindstick/AutoFlashlightManager.kt new file mode 100644 index 0000000..82576b5 --- /dev/null +++ b/app/src/main/java/com/example/smartblindstick/AutoFlashlightManager.kt @@ -0,0 +1,87 @@ +package com.example.smartblindstick + +import android.content.Context +import android.hardware.Sensor +import android.hardware.SensorEvent +import android.hardware.SensorEventListener +import android.hardware.SensorManager +import android.util.Log +import androidx.camera.core.Camera + +class AutoFlashlightManager( + private val context: Context, + private val onSpeak: (String) -> Unit +) { + + private var camera: Camera? = null + private var isTorchOn = false + + // Debouncing/Hysteresis state + private var lowLightCount = 0 + private var brightLightCount = 0 + // Based on typical Y-plane luminance (0-255) + private val THRESHOLD_LOW = 80f // Turn on sooner in darkness + private val THRESHOLD_BRIGHT = 200f // Only turn off when very bright + private val REQUIRED_CONSECUTIVE_READINGS_ON = 1 // Instantly turn on + private val REQUIRED_CONSECUTIVE_READINGS_OFF = 5 // Be sure before turning off + + var isManualOverride = false + + fun setCamera(camera: Camera?) { + this.camera = camera + } + + fun startMonitoring() { + Log.d("AutoFlashlight", "Started camera luminance monitoring") + } + + fun stopMonitoring() { + setTorch(false) + } + + fun processLuminance(averageLuminance: Float) { + if (camera == null || !camera!!.cameraInfo.hasFlashUnit() || isManualOverride) return + + if (averageLuminance < THRESHOLD_LOW && !isTorchOn) { + lowLightCount++ + brightLightCount = 0 + if (lowLightCount >= REQUIRED_CONSECUTIVE_READINGS_ON) { + setTorch(true) + onSpeak("Low light detected. Flashlight auto-enabled.") + lowLightCount = 0 + } + } else if (averageLuminance > THRESHOLD_BRIGHT && isTorchOn) { + brightLightCount++ + lowLightCount = 0 + if (brightLightCount >= REQUIRED_CONSECUTIVE_READINGS_OFF) { + setTorch(false) + brightLightCount = 0 + } + } else { + // Reset counters if reading is in between thresholds + lowLightCount = 0 + brightLightCount = 0 + } + } + + fun toggleTorch(enable: Boolean) { + isManualOverride = true + setTorch(enable) + if (enable) onSpeak("Flashlight manually turned on. Auto mode paused.") else onSpeak("Flashlight manually turned off. Auto mode paused.") + } + + fun resetToAuto() { + isManualOverride = false + onSpeak("Auto flashlight resumed.") + } + + private fun setTorch(enable: Boolean) { + try { + camera?.cameraControl?.enableTorch(enable) + isTorchOn = enable + Log.d("AutoFlashlight", "Torch set to $enable") + } catch (e: Exception) { + Log.e("AutoFlashlight", "Failed to set torch", e) + } + } +} diff --git a/app/src/main/java/com/example/smartblindstick/CompassManager.kt b/app/src/main/java/com/example/smartblindstick/CompassManager.kt new file mode 100644 index 0000000..2910cc2 --- /dev/null +++ b/app/src/main/java/com/example/smartblindstick/CompassManager.kt @@ -0,0 +1,38 @@ +package com.example.smartblindstick + +import android.content.Context +import android.hardware.Sensor +import android.hardware.SensorEvent +import android.hardware.SensorEventListener +import android.hardware.SensorManager + +class CompassManager(context: Context) : SensorEventListener { + + private val sensorManager = context.getSystemService(Context.SENSOR_SERVICE) as SensorManager + private val rotationSensor = sensorManager.getDefaultSensor(Sensor.TYPE_ROTATION_VECTOR) + + @Volatile + var currentAzimuth: Float = 0f + + fun start() { + rotationSensor?.let { + sensorManager.registerListener(this, it, SensorManager.SENSOR_DELAY_UI) + } + } + + fun stop() { + sensorManager.unregisterListener(this) + } + + override fun onSensorChanged(event: SensorEvent) { + if (event.sensor.type == Sensor.TYPE_ROTATION_VECTOR) { + val rotationMatrix = FloatArray(9) + SensorManager.getRotationMatrixFromVector(rotationMatrix, event.values) + val orientation = FloatArray(3) + SensorManager.getOrientation(rotationMatrix, orientation) + currentAzimuth = (Math.toDegrees(orientation[0].toDouble()) + 360).toFloat() % 360 + } + } + + override fun onAccuracyChanged(sensor: Sensor?, accuracy: Int) {} +} diff --git a/app/src/main/java/com/example/smartblindstick/FallDetectionManager.kt b/app/src/main/java/com/example/smartblindstick/FallDetectionManager.kt new file mode 100644 index 0000000..80e43f6 --- /dev/null +++ b/app/src/main/java/com/example/smartblindstick/FallDetectionManager.kt @@ -0,0 +1,165 @@ +package com.example.smartblindstick + +import android.content.Context +import android.hardware.Sensor +import android.hardware.SensorEvent +import android.hardware.SensorEventListener +import android.hardware.SensorManager +import android.telephony.SmsManager +import android.util.Log +import kotlin.math.sqrt +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody +import android.location.LocationManager +import android.Manifest +import android.content.pm.PackageManager +import androidx.core.content.ContextCompat +import org.json.JSONObject + +class FallDetectionManager(private val context: Context) : SensorEventListener { + + private val okHttpClient = OkHttpClient() + private var sensorManager: SensorManager? = null + private var accelerometer: Sensor? = null + + // Fall detection state variables + private var isFreeFallDetected = false + private var lastFreeFallTime: Long = 0 + + // Thresholds + private val FREE_FALL_THRESHOLD = 3.0f // m/s^2 (near zero gravity) + private val IMPACT_THRESHOLD = 20.0f // m/s^2 (hard impact) + + // Emergency contact (Change this or build UI to set it) + private val EMERGENCY_NUMBER = "1234567890" + + init { + sensorManager = context.getSystemService(Context.SENSOR_SERVICE) as SensorManager + accelerometer = sensorManager?.getDefaultSensor(Sensor.TYPE_ACCELEROMETER) + } + + fun startListening() { + accelerometer?.let { + sensorManager?.registerListener(this, it, SensorManager.SENSOR_DELAY_NORMAL) + Log.d("FallDetection", "Started listening to accelerometer") + } + } + + fun stopListening() { + sensorManager?.unregisterListener(this) + } + + override fun onSensorChanged(event: SensorEvent?) { + if (event == null || event.sensor.type != Sensor.TYPE_ACCELEROMETER) return + + val x = event.values[0] + val y = event.values[1] + val z = event.values[2] + + // Calculate total acceleration magnitude + val acceleration = sqrt((x * x + y * y + z * z).toDouble()).toFloat() + + // 1. Detect Free Fall + if (acceleration < FREE_FALL_THRESHOLD) { + isFreeFallDetected = true + lastFreeFallTime = System.currentTimeMillis() + } + + // 2. Detect Impact shortly after Free Fall + if (isFreeFallDetected && acceleration > IMPACT_THRESHOLD) { + val timeSinceFreeFall = System.currentTimeMillis() - lastFreeFallTime + + // If impact happens within 2 seconds of free fall, it's likely a drop/fall + if (timeSinceFreeFall < 2000) { + Log.w("FallDetection", "FALL DETECTED! Acceleration: $acceleration") + triggerEmergencyProtocol() + + // Reset state + isFreeFallDetected = false + } + } + + // Reset free fall if too much time passes + if (isFreeFallDetected && System.currentTimeMillis() - lastFreeFallTime > 2000) { + isFreeFallDetected = false + } + } + + private fun triggerEmergencyProtocol() { + // Speak alert using MainActivity reference + if (context is MainActivity) { + context.speak("Fall detected. Sending emergency message.") + } + + sendTelegramAlert() + } + + private fun sendTelegramAlert() { + // Real credentials for user + val botToken = "8654874065:AAGFOscZtCQUOokXO7QsvFbuBbWeb8ltdU4" + val chatId = "7010211918" + + var locationString = "Location unavailable." + try { + if (ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) == PackageManager.PERMISSION_GRANTED || + ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_COARSE_LOCATION) == PackageManager.PERMISSION_GRANTED) { + + val locationManager = context.getSystemService(Context.LOCATION_SERVICE) as LocationManager + var lastLoc = locationManager.getLastKnownLocation(LocationManager.GPS_PROVIDER) + if (lastLoc == null) { + lastLoc = locationManager.getLastKnownLocation(LocationManager.NETWORK_PROVIDER) + } + if (lastLoc != null) { + locationString = "Location: https://www.google.com/maps/search/?api=1&query=${lastLoc.latitude},${lastLoc.longitude}" + } + } + } catch (e: Exception) { + Log.e("FallDetection", "Error getting location", e) + } + + val message = "Emergency! The Smart Blind Stick user may have fallen. Please check on them.\n$locationString" + + if (botToken == "") { + Log.w("FallDetection", "Telegram Bot Token not set. Cannot send alert.") + return + } + + val url = "https://api.telegram.org/bot$botToken/sendMessage" + + // Build JSON body + val json = JSONObject() + json.put("chat_id", chatId) + json.put("text", message) + + val body = RequestBody.create( + "application/json; charset=utf-8".toMediaTypeOrNull(), + json.toString() + ) + + val request = Request.Builder() + .url(url) + .post(body) + .build() + + // Use Coroutine to send network request off the main thread + CoroutineScope(Dispatchers.IO).launch { + try { + val response = okHttpClient.newCall(request).execute() + if (response.isSuccessful) { + Log.d("FallDetection", "Telegram emergency message sent successfully") + } else { + Log.e("FallDetection", "Failed to send Telegram message: ${response.code}") + } + } catch (e: Exception) { + Log.e("FallDetection", "Error sending Telegram message", e) + } + } + } + + override fun onAccuracyChanged(sensor: Sensor?, accuracy: Int) {} +} diff --git a/app/src/main/java/com/example/smartblindstick/MainActivity.kt b/app/src/main/java/com/example/smartblindstick/MainActivity.kt new file mode 100644 index 0000000..2c923c2 --- /dev/null +++ b/app/src/main/java/com/example/smartblindstick/MainActivity.kt @@ -0,0 +1,466 @@ +package com.example.smartblindstick + +import android.Manifest +import android.content.pm.PackageManager +import android.os.Bundle +import android.speech.tts.TextToSpeech +import android.util.Log +import android.widget.Toast +import androidx.activity.ComponentActivity +import androidx.activity.compose.setContent +import androidx.activity.result.contract.ActivityResultContracts +import androidx.camera.core.CameraSelector +import androidx.camera.core.ImageAnalysis +import androidx.camera.core.Camera +import androidx.camera.core.Preview +import androidx.camera.lifecycle.ProcessCameraProvider +import androidx.camera.view.PreviewView +import androidx.compose.foundation.clickable +import androidx.compose.foundation.Canvas +import androidx.compose.foundation.layout.* +import androidx.compose.material3.* +import androidx.compose.runtime.* +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.geometry.Size +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.drawscope.Stroke +import androidx.compose.ui.graphics.nativeCanvas +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.LocalLifecycleOwner +import androidx.compose.ui.unit.dp +import androidx.compose.ui.viewinterop.AndroidView +import androidx.core.content.ContextCompat +import com.example.smartblindstick.ui.theme.SmartBlindStickTheme +import java.util.Locale +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +class MainActivity : ComponentActivity(), TextToSpeech.OnInitListener { + + private lateinit var cameraExecutor: ExecutorService + private lateinit var tts: TextToSpeech + private lateinit var compassManager: CompassManager + + private lateinit var fallDetectionManager: FallDetectionManager + private lateinit var navigationManager: NavigationManager + private lateinit var visionProcessor: VisionProcessor + private lateinit var voiceAssistantManager: VoiceAssistantManager + private lateinit var ocrProcessor: OcrProcessor + + // New feature managers + private lateinit var spatialAudioManager: SpatialAudioManager + private lateinit var visualMemoryManager: VisualMemoryManager + private lateinit var microNavigationManager: MicroNavigationManager + private lateinit var autoFlashlightManager: AutoFlashlightManager + + @Volatile + private var isOcrRequested = false + private var ocrRequestTime = 0L + @Volatile + private var isNvidiaRequested = false + @Volatile + private var nvidiaPrompt = "" + + private val REQUIRED_PERMISSIONS = arrayOf( + Manifest.permission.CAMERA, + Manifest.permission.RECORD_AUDIO, + Manifest.permission.VIBRATE, + Manifest.permission.ACCESS_FINE_LOCATION, + Manifest.permission.ACCESS_COARSE_LOCATION + ) + + private val requestPermissionLauncher = registerForActivityResult( + ActivityResultContracts.RequestMultiplePermissions() + ) { permissions -> + var allGranted = true + permissions.entries.forEach { + if (!it.value) allGranted = false + } + if (allGranted) { + fallDetectionManager.startListening() + } else { + Toast.makeText(this, "Permissions not granted by the user.", Toast.LENGTH_SHORT).show() + } + } + + private var pendingCsvContent: String? = null + private val createDocumentLauncher = registerForActivityResult( + ActivityResultContracts.StartActivityForResult() + ) { result -> + if (result.resultCode == android.app.Activity.RESULT_OK) { + result.data?.data?.let { uri -> + pendingCsvContent?.let { content -> + try { + contentResolver.openOutputStream(uri)?.use { outputStream -> + outputStream.write(content.toByteArray()) + } + speak("Metadata downloaded successfully.", isHighPriority = true) + } catch (e: Exception) { + Log.e("MainActivity", "Error saving CSV", e) + speak("Failed to save metadata.", isHighPriority = true) + } + } + } + } + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + + tts = TextToSpeech(this, this) + fallDetectionManager = FallDetectionManager(this) + navigationManager = NavigationManager(this) + visionProcessor = VisionProcessor(this) + ocrProcessor = OcrProcessor(this) + + compassManager = CompassManager(this) + spatialAudioManager = SpatialAudioManager { msg, pan, vol, pitch -> speak(msg, pan, vol, pitch, false) } + visualMemoryManager = VisualMemoryManager(this) { msg, pan, vol, pitch -> speak(msg, pan, vol, pitch, true) } + visualMemoryManager.compassManager = compassManager + microNavigationManager = MicroNavigationManager(this) { msg -> speak(msg, isHighPriority = false) } + autoFlashlightManager = AutoFlashlightManager(this) { msg -> speak(msg, isHighPriority = false) } + + visionProcessor.spatialAudioManager = spatialAudioManager + visionProcessor.visualMemoryManager = visualMemoryManager + visionProcessor.microNavigationManager = microNavigationManager + visionProcessor.autoFlashlightManager = autoFlashlightManager + + autoFlashlightManager.startMonitoring() + compassManager.start() + + voiceAssistantManager = VoiceAssistantManager(this) { command -> + runOnUiThread { + handleVoiceCommand(command) + } + } + + cameraExecutor = Executors.newSingleThreadExecutor() + + if (allPermissionsGranted()) { + fallDetectionManager.startListening() + } else { + requestPermissionLauncher.launch(REQUIRED_PERMISSIONS) + } + + setContent { + SmartBlindStickTheme { + Surface( + modifier = Modifier.fillMaxSize(), + color = MaterialTheme.colorScheme.background + ) { + MainScreen( + cameraExecutor = cameraExecutor, + visionProcessor = visionProcessor, + onAnalyzeImage = { imageProxy -> + if (isOcrRequested) { + ocrProcessor.processImageProxy(imageProxy) { detectedText -> + isOcrRequested = false + speak("I read: $detectedText", isHighPriority = true) + } + if (System.currentTimeMillis() - ocrRequestTime > 5000 && isOcrRequested) { + isOcrRequested = false + speak("I couldn't find any clear text.", isHighPriority = true) + } + } else if (isNvidiaRequested) { + isNvidiaRequested = false + val currentPrompt = nvidiaPrompt + visionProcessor.processForNvidia(imageProxy, currentPrompt) { answer -> + speak(answer, isHighPriority = true) + } + } else { + visionProcessor.processImageProxy(imageProxy) + } + }, + onCameraBound = { camera -> + autoFlashlightManager.setCamera(camera) + }, + onMicClick = { + tts.stop() + voiceAssistantManager.startListening() + } + ) + } + } + } + } + + override fun onInit(status: Int) { + if (status == TextToSpeech.SUCCESS) { + val result = tts.setLanguage(Locale.US) + if (result == TextToSpeech.LANG_MISSING_DATA || result == TextToSpeech.LANG_NOT_SUPPORTED) { + Log.e("TTS", "Language not supported") + } else { + speak("Smart Blind Stick system initialized") + // Do not auto-start listening anymore + } + } + } + + private fun handleVoiceCommand(command: String) { + when { + command.contains("read") -> { + isOcrRequested = true + ocrRequestTime = System.currentTimeMillis() + speak("Let me read that for you.", isHighPriority = true) + } + command.contains("stop") -> { + tts.stop() + speak("Stopped.", isHighPriority = true) + } + command.contains("turn on flashlight") || command.contains("turn flashlight on") || command.contains("torch on") -> { + autoFlashlightManager.toggleTorch(true) + } + command.contains("turn off flashlight") || command.contains("turn flashlight off") || command.contains("torch off") -> { + autoFlashlightManager.toggleTorch(false) + } + command.contains("auto flashlight") || command.contains("automatic flashlight") -> { + autoFlashlightManager.resetToAuto() + } + command.contains("ask nvidia") || command.contains("nvidia") || command.contains("describe") -> { + val prompt = command.replace("ask nvidia", "").replace("nvidia", "").trim() + nvidiaPrompt = if (prompt.isEmpty()) "Describe what you see in front of me." else prompt + isNvidiaRequested = true + speak("Asking Nvidia...", isHighPriority = true) + } + command.contains("go to") || command.contains("navigate to") -> { + val destination = command.replace("go to", "").replace("navigate to", "").trim() + if (destination.isNotEmpty()) { + speak("Starting navigation to $destination", isHighPriority = true) + navigationManager.startNavigation(destination) + } else { + speak("Please specify a destination.", isHighPriority = true) + } + } + command.contains("switch to spatial") || command.contains("spacial") || command.contains("special") -> { + spatialAudioManager.setEnabled(true) + speak("Spatial audio enabled", isHighPriority = true) + } + command.contains("switch to voice") || command.contains("voice") -> { + spatialAudioManager.setEnabled(false) + speak("Spatial audio disabled, voice enabled", isHighPriority = true) + } + command.contains("where is my") || command.contains("where did i put my") -> { + speak("Let me check my visual memory.", isHighPriority = true) + visualMemoryManager.answerQuery(command) + } + command.contains("export metadata") || command.contains("extract metadata") -> { + speak("Preparing metadata file. Please choose where to save it on your screen.", isHighPriority = true) + visualMemoryManager.exportMetadata { csvContent -> + val intent = android.content.Intent(android.content.Intent.ACTION_CREATE_DOCUMENT).apply { + addCategory(android.content.Intent.CATEGORY_OPENABLE) + type = "text/csv" + putExtra(android.content.Intent.EXTRA_TITLE, "SmartBlindStick_Metadata.csv") + } + pendingCsvContent = csvContent + createDocumentLauncher.launch(intent) + } + } + command.contains("transit mode") || command.contains("find a door") -> { + microNavigationManager.currentMode = NavMode.TRANSIT + nvidiaPrompt = "I am looking for a door or an exit. Tell me exactly where it is using a 3x3 grid (e.g. 'door is center'). If you don't see one, say 'No doors in current view'." + isNvidiaRequested = true + speak("Scanning for doors and transit vehicles...", isHighPriority = true) + } + command.contains("find a seat") || command.contains("find a bed") -> { + microNavigationManager.currentMode = NavMode.SEAT + nvidiaPrompt = "Differentiate between any beds, chairs, couches, or doors in this view. Tell me their exact locations using a 3x3 grid (e.g. 'bed is bottom left', 'door is center')." + isNvidiaRequested = true + speak("Scanning room layout for seats, beds, and doors...", isHighPriority = true) + } + command.contains("find a cup") || command.contains("find a bottle") -> { + microNavigationManager.currentMode = NavMode.CUP + } + command.startsWith("find ") -> { + val target = command.replace("find ", "").trim() + if (target.isNotEmpty()) { + microNavigationManager.startFinding(target) + + // Trigger cloud Vision AI for advanced object recognition + nvidiaPrompt = "I am looking for a $target. Do you see it in this image? If so, tell me exactly where it is using a 3x3 grid (e.g. 'top left', 'center', 'bottom right') and provide a brief description. If you don't see it, reply EXACTLY with 'Not found in current view'." + isNvidiaRequested = true + speak("Scanning for $target...", isHighPriority = true) + } + } + command.contains("normal mode") -> { + microNavigationManager.currentMode = NavMode.NORMAL + } + else -> { + speak("Command not recognized. Please try again.", isHighPriority = true) + } + } + } + + private var currentSpeechIsHighPriority = false + + fun speak(text: String, pan: Float = 0f, volume: Float = 1.0f, pitch: Float = 1.0f, isHighPriority: Boolean = false) { + if (tts.isSpeaking && currentSpeechIsHighPriority && !isHighPriority) { + return // Don't allow background tracking to interrupt high-priority AI answers + } + + currentSpeechIsHighPriority = isHighPriority + val params = Bundle() + params.putFloat(TextToSpeech.Engine.KEY_PARAM_PAN, pan.coerceIn(-1.0f, 1.0f)) + params.putFloat(TextToSpeech.Engine.KEY_PARAM_VOLUME, volume.coerceIn(0.0f, 1.0f)) + tts.setPitch(pitch.coerceIn(0.5f, 2.0f)) + tts.speak(text, TextToSpeech.QUEUE_FLUSH, params, "") + } + + private fun allPermissionsGranted() = REQUIRED_PERMISSIONS.all { + ContextCompat.checkSelfPermission(baseContext, it) == PackageManager.PERMISSION_GRANTED + } + + override fun onDestroy() { + super.onDestroy() + cameraExecutor.shutdown() + tts.stop() + tts.shutdown() + fallDetectionManager.stopListening() + voiceAssistantManager.destroy() + autoFlashlightManager.stopMonitoring() + compassManager.stop() + } +} + +@Composable +fun MainScreen( + cameraExecutor: ExecutorService, + visionProcessor: VisionProcessor, + onAnalyzeImage: (androidx.camera.core.ImageProxy) -> Unit, + onCameraBound: (Camera) -> Unit, + onMicClick: () -> Unit +) { + var lensFacing by remember { mutableIntStateOf(CameraSelector.LENS_FACING_BACK) } + var currentDetections by remember { mutableStateOf>(emptyList()) } + var imageDimensions by remember { mutableStateOf(Pair(1, 1)) } + + DisposableEffect(visionProcessor) { + visionProcessor.onDetectionsUpdated = { detections, width, height -> + currentDetections = detections + imageDimensions = Pair(width, height) + } + onDispose { + visionProcessor.onDetectionsUpdated = null + } + } + + Box(modifier = Modifier.fillMaxSize()) { + Box( + modifier = Modifier + .fillMaxWidth() + .aspectRatio(3f / 4f) + .align(Alignment.Center) + .clickable { onMicClick() } + ) { + CameraPreviewFeed( + cameraExecutor = cameraExecutor, + onAnalyzeImage = onAnalyzeImage, + onCameraBound = onCameraBound, + lensFacing = lensFacing, + modifier = Modifier.fillMaxSize() + ) + + Canvas(modifier = Modifier.fillMaxSize()) { + val scaleX = size.width / imageDimensions.first.toFloat() + val scaleY = size.height / imageDimensions.second.toFloat() + + val paint = android.graphics.Paint().apply { + color = android.graphics.Color.GREEN + textSize = 60f + isFakeBoldText = true + } + + for (det in currentDetections) { + val left = det.boundingBox.left * scaleX + val top = det.boundingBox.top * scaleY + val right = det.boundingBox.right * scaleX + val bottom = det.boundingBox.bottom * scaleY + + drawRect( + color = Color.Green, + topLeft = Offset(left, top), + size = Size(right - left, bottom - top), + style = Stroke(width = 8f) + ) + + drawContext.canvas.nativeCanvas.drawText( + det.label, left, top - 10f, paint + ) + } + } + } + + Row( + modifier = Modifier + .align(Alignment.BottomCenter) + .padding(bottom = 32.dp), + horizontalArrangement = Arrangement.spacedBy(16.dp) + ) { + Button(onClick = onMicClick) { + Text("Tap to Speak") + } + Button(onClick = { + lensFacing = if (lensFacing == CameraSelector.LENS_FACING_BACK) { + CameraSelector.LENS_FACING_FRONT + } else { + CameraSelector.LENS_FACING_BACK + } + }) { + Text("Flip Camera") + } + } + } +} + +@Composable +fun CameraPreviewFeed( + cameraExecutor: ExecutorService, + onAnalyzeImage: (androidx.camera.core.ImageProxy) -> Unit, + onCameraBound: (Camera) -> Unit, + lensFacing: Int, + modifier: Modifier = Modifier +) { + val context = LocalContext.current + val lifecycleOwner = LocalLifecycleOwner.current + + AndroidView( + factory = { ctx -> + PreviewView(ctx) + }, + update = { previewView -> + val cameraProviderFuture = ProcessCameraProvider.getInstance(context) + cameraProviderFuture.addListener({ + val cameraProvider = cameraProviderFuture.get() + + val preview = Preview.Builder() + .build() + .also { + it.setSurfaceProvider(previewView.surfaceProvider) + } + + val imageAnalyzer = ImageAnalysis.Builder() + .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) + .build() + .also { + it.setAnalyzer(cameraExecutor) { imageProxy -> + onAnalyzeImage(imageProxy) + } + } + + val cameraSelector = CameraSelector.Builder().requireLensFacing(lensFacing).build() + + try { + cameraProvider.unbindAll() + val camera = cameraProvider.bindToLifecycle( + lifecycleOwner, cameraSelector, preview, imageAnalyzer + ) + onCameraBound(camera) + } catch (exc: Exception) { + Log.e("CameraX", "Use case binding failed", exc) + } + }, ContextCompat.getMainExecutor(context)) + }, + modifier = modifier + ) +} \ No newline at end of file diff --git a/app/src/main/java/com/example/smartblindstick/MetaApiClient.kt b/app/src/main/java/com/example/smartblindstick/MetaApiClient.kt new file mode 100644 index 0000000..40bfa79 --- /dev/null +++ b/app/src/main/java/com/example/smartblindstick/MetaApiClient.kt @@ -0,0 +1,65 @@ +package com.example.smartblindstick + +import android.util.Log +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody +import org.json.JSONArray +import org.json.JSONObject + +class MetaApiClient { + private val client = OkHttpClient() + private val apiKey = "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" + // Assuming NVIDIA NIM or a similar OpenAI-compatible endpoint format for this key pattern + private val apiUrl = "https://integrate.api.nvidia.com/v1/chat/completions" + + suspend fun generateResponse(prompt: String): String? { + return withContext(Dispatchers.IO) { + try { + val jsonBody = JSONObject().apply { + put("model", "meta/llama-3.1-70b-instruct") // Using a high quality text model assuming NVIDIA endpoint + val messages = JSONArray() + messages.put(JSONObject().apply { + put("role", "user") + put("content", prompt) + }) + put("messages", messages) + put("max_tokens", 100) + put("temperature", 0.5) + } + + val requestBody = RequestBody.create( + "application/json; charset=utf-8".toMediaTypeOrNull(), + jsonBody.toString() + ) + + val request = Request.Builder() + .url(apiUrl) + .addHeader("Authorization", "Bearer $apiKey") + .addHeader("Accept", "application/json") + .post(requestBody) + .build() + + val response = client.newCall(request).execute() + if (response.isSuccessful) { + val responseBody = response.body?.string() ?: return@withContext null + val jsonResponse = JSONObject(responseBody) + val choices = jsonResponse.getJSONArray("choices") + if (choices.length() > 0) { + val firstChoice = choices.getJSONObject(0) + val message = firstChoice.getJSONObject("message") + return@withContext message.getString("content") + } + } else { + Log.e("MetaApiClient", "API request failed with code: ${response.code}, body: ${response.body?.string()}") + } + } catch (e: Exception) { + Log.e("MetaApiClient", "Error calling Meta API", e) + } + null + } + } +} diff --git a/app/src/main/java/com/example/smartblindstick/MicroNavigationManager.kt b/app/src/main/java/com/example/smartblindstick/MicroNavigationManager.kt new file mode 100644 index 0000000..3ae2204 --- /dev/null +++ b/app/src/main/java/com/example/smartblindstick/MicroNavigationManager.kt @@ -0,0 +1,253 @@ +package com.example.smartblindstick + +import android.content.Context +import android.os.Build +import android.os.VibrationEffect +import android.os.Vibrator +import android.os.VibratorManager +import android.util.Log + +enum class NavMode { + NORMAL, TRANSIT, SEAT, CUP, FIND +} + +class MicroNavigationManager(private val context: Context, private val onSpeak: (String) -> Unit) { + + var currentMode: NavMode = NavMode.NORMAL + set(value) { + field = value + when (value) { + NavMode.TRANSIT -> onSpeak("Transit mode activated. Looking for doors.") + NavMode.SEAT -> onSpeak("Seat mode activated. Looking for empty seats.") + NavMode.CUP -> onSpeak("Cup mode activated. Looking for cups and bottles.") + NavMode.NORMAL -> onSpeak("Micro navigation returned to normal mode.") + NavMode.FIND -> {} // Handled by startFinding + } + } + + private var searchTarget: String = "" + + fun startFinding(target: String) { + searchTarget = target.lowercase() + currentMode = NavMode.FIND + onSpeak("Looking for $searchTarget") + } + + private val vibrator: Vibrator = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + val vibratorManager = context.getSystemService(Context.VIBRATOR_MANAGER_SERVICE) as VibratorManager + vibratorManager.defaultVibrator + } else { + @Suppress("DEPRECATION") + context.getSystemService(Context.VIBRATOR_SERVICE) as Vibrator + } + + private var lastAnnouncementTime = 0L + private var lastVibrationTime = 0L + private var lastSpokenTarget: String = "" + + private fun attemptAnnouncement(announcement: String, distanceStr: String) { + if (distanceStr != "close") return + + val currentTime = System.currentTimeMillis() + if (announcement != lastSpokenTarget || currentTime - lastAnnouncementTime > 15000) { + onSpeak(announcement) + lastSpokenTarget = announcement + lastAnnouncementTime = currentTime + } + } + + fun processDetections(detections: List, imageWidth: Int, imageHeight: Int) { + when (currentMode) { + NavMode.NORMAL -> processNormalMode(detections, imageWidth, imageHeight) + NavMode.TRANSIT -> processTransitMode(detections, imageWidth, imageHeight) + NavMode.SEAT -> processSeatMode(detections, imageWidth, imageHeight) + NavMode.CUP -> processCupMode(detections, imageWidth, imageHeight) + NavMode.FIND -> processFindMode(detections, imageWidth, imageHeight) + } + } + + private fun processNormalMode(detections: List, imageWidth: Int, imageHeight: Int) { + val closeObjects = detections.filter { it.distanceMeters in 0f..1.0f } + if (closeObjects.isEmpty()) return + + val largest = closeObjects.maxByOrNull { it.boundingBox.width() * it.boundingBox.height() } ?: return + val currentTime = System.currentTimeMillis() + if (currentTime - lastVibrationTime > 2000) { + triggerVibration(getPositionDescription(largest.centerX, largest.centerY, imageWidth, imageHeight)) + lastVibrationTime = currentTime + } + } + + private fun processFindMode(detections: List, imageWidth: Int, imageHeight: Int) { + val targets = detections.filter { + val label = it.label.lowercase() + label.contains(searchTarget) || searchTarget.contains(label) + } + if (targets.isEmpty()) return + + val target = targets.maxByOrNull { it.boundingBox.width() * it.boundingBox.height() } ?: return + + val position = getPositionDescription(target.centerX, target.centerY, imageWidth, imageHeight) + val distanceStr = getDistanceDescription(target.distanceMeters) + attemptAnnouncement("${target.label} detected $position, $distanceStr.", distanceStr) + + val currentTime = System.currentTimeMillis() + if (currentTime - lastVibrationTime > 1000) { + triggerVibration(position) + lastVibrationTime = currentTime + } + } + + private fun processTransitMode(detections: List, imageWidth: Int, imageHeight: Int) { + val vehicles = detections.filter { it.label in listOf("bus", "train") } + if (vehicles.isEmpty()) return + + val largest = vehicles.maxByOrNull { it.boundingBox.width() * it.boundingBox.height() } ?: return + + val position = getPositionDescription(largest.centerX, largest.centerY, imageWidth, imageHeight) + val distanceStr = getDistanceDescription(largest.distanceMeters) + attemptAnnouncement("Transit vehicle detected $position, $distanceStr.", distanceStr) + + // Haptic feedback + val currentTime = System.currentTimeMillis() + if (currentTime - lastVibrationTime > 1000) { + triggerVibration(getPositionDescription(largest.centerX, largest.centerY, imageWidth, imageHeight)) + lastVibrationTime = currentTime + } + } + + private fun processSeatMode(detections: List, imageWidth: Int, imageHeight: Int) { + val chairs = detections.filter { it.label in listOf("chair", "bench", "couch", "bed") } + val people = detections.filter { it.label == "person" } + + if (chairs.isEmpty()) return + + for (chair in chairs) { + var isOccupied = false + for (person in people) { + if (calculateIoU(chair.boundingBox, person.boundingBox) > 0.1f) { + isOccupied = true + break + } + } + + if (!isOccupied) { + val position = getPositionDescription(chair.centerX, chair.centerY, imageWidth, imageHeight) + val distanceStr = getDistanceDescription(chair.distanceMeters) + attemptAnnouncement("Empty seat detected $position, $distanceStr.", distanceStr) + } + } + + // Haptic feedback for empty seats + val currentTime = System.currentTimeMillis() + for (chair in chairs) { + var isOccupied = false + for (person in people) { + if (calculateIoU(chair.boundingBox, person.boundingBox) > 0.1f) { + isOccupied = true + break + } + } + if (!isOccupied && currentTime - lastVibrationTime > 1000) { + triggerVibration(getPositionDescription(chair.centerX, chair.centerY, imageWidth, imageHeight)) + lastVibrationTime = currentTime + return + } + } + } + + private fun processCupMode(detections: List, imageWidth: Int, imageHeight: Int) { + val targets = detections.filter { it.label in listOf("cup", "bottle", "wine glass") } + if (targets.isEmpty()) return + + val target = targets.maxByOrNull { it.boundingBox.width() * it.boundingBox.height() } ?: return + + val position = getPositionDescription(target.centerX, target.centerY, imageWidth, imageHeight) + val distanceStr = getDistanceDescription(target.distanceMeters) + attemptAnnouncement("${target.label} detected $position, $distanceStr.", distanceStr) + + val currentTime = System.currentTimeMillis() + if (currentTime - lastVibrationTime > 1000) { + triggerVibration(position) + lastVibrationTime = currentTime + } + } + + private fun triggerVibration(position: String) { + when { + position.contains("left") -> vibrateLeft() + position.contains("right") -> vibrateRight() + else -> vibrateCenter() + } + } + + private fun getDistanceDescription(distanceMeters: Float): String { + return when { + distanceMeters < 0 -> "" + distanceMeters < 1.0f -> "close" + distanceMeters < 2.5f -> "medium" + else -> "far" + } + } + + private fun getPositionDescription(centerX: Float, centerY: Float, imageWidth: Int, imageHeight: Int): String { + val hPos = when { + centerX < imageWidth / 3 -> "left" + centerX > 2 * imageWidth / 3 -> "right" + else -> "center" + } + val vPos = when { + centerY < imageHeight / 3 -> "top" + centerY > 2 * imageHeight / 3 -> "bottom" + else -> "middle" + } + return if (hPos == "center" && vPos == "middle") "center" else "$vPos $hPos" + } + + private fun vibrateLeft() { + // Short then long + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + vibrator.vibrate(VibrationEffect.createWaveform(longArrayOf(0, 100, 100, 300), -1)) + } else { + @Suppress("DEPRECATION") + vibrator.vibrate(longArrayOf(0, 100, 100, 300), -1) + } + } + + private fun vibrateRight() { + // Long then short + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + vibrator.vibrate(VibrationEffect.createWaveform(longArrayOf(0, 300, 100, 100), -1)) + } else { + @Suppress("DEPRECATION") + vibrator.vibrate(longArrayOf(0, 300, 100, 100), -1) + } + } + + private fun vibrateCenter() { + // Solid pulse + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + vibrator.vibrate(VibrationEffect.createOneShot(200, VibrationEffect.DEFAULT_AMPLITUDE)) + } else { + @Suppress("DEPRECATION") + vibrator.vibrate(200) + } + } + + private fun calculateIoU(box1: android.graphics.RectF, box2: android.graphics.RectF): Float { + val intersectionLeft = maxOf(box1.left, box2.left) + val intersectionTop = maxOf(box1.top, box2.top) + val intersectionRight = minOf(box1.right, box2.right) + val intersectionBottom = minOf(box1.bottom, box2.bottom) + + if (intersectionRight < intersectionLeft || intersectionBottom < intersectionTop) { + return 0.0f + } + + val intersectionArea = (intersectionRight - intersectionLeft) * (intersectionBottom - intersectionTop) + val box1Area = box1.width() * box1.height() + val box2Area = box2.width() * box2.height() + + return intersectionArea / (box1Area + box2Area - intersectionArea) + } +} diff --git a/app/src/main/java/com/example/smartblindstick/NavigationManager.kt b/app/src/main/java/com/example/smartblindstick/NavigationManager.kt new file mode 100644 index 0000000..4484d6a --- /dev/null +++ b/app/src/main/java/com/example/smartblindstick/NavigationManager.kt @@ -0,0 +1,31 @@ +package com.example.smartblindstick + +import android.content.Context +import android.content.Intent +import android.net.Uri +import android.util.Log + +class NavigationManager(private val context: Context) { + + fun startNavigation(destination: String) { + // Create a Uri for Google Maps Navigation + // Format: google.navigation:q=latitude,longitude OR google.navigation:q=Address + val gmmIntentUri = Uri.parse("google.navigation:q=${Uri.encode(destination)}&mode=w") // mode=w is for walking + + // Create an Intent from gmmIntentUri + val mapIntent = Intent(Intent.ACTION_VIEW, gmmIntentUri) + + // Make the Intent explicit by setting the Google Maps package + mapIntent.setPackage("com.google.android.apps.maps") + + try { + context.startActivity(mapIntent) + Log.d("Navigation", "Started Google Maps navigation to $destination") + } catch (e: Exception) { + Log.e("Navigation", "Google Maps app is not installed or couldn't handle intent.") + if (context is MainActivity) { + context.speak("Google Maps is not installed on this device.") + } + } + } +} diff --git a/app/src/main/java/com/example/smartblindstick/NvidiaApiClient.kt b/app/src/main/java/com/example/smartblindstick/NvidiaApiClient.kt new file mode 100644 index 0000000..a5df611 --- /dev/null +++ b/app/src/main/java/com/example/smartblindstick/NvidiaApiClient.kt @@ -0,0 +1,107 @@ +package com.example.smartblindstick + +import android.graphics.Bitmap +import android.util.Base64 +import android.util.Log +import okhttp3.* +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.RequestBody.Companion.toRequestBody +import org.json.JSONArray +import org.json.JSONObject +import java.io.ByteArrayOutputStream +import java.io.IOException +import java.util.concurrent.TimeUnit + +class NvidiaApiClient { + private val client = OkHttpClient.Builder() + .connectTimeout(10, TimeUnit.SECONDS) + .readTimeout(10, TimeUnit.SECONDS) + .writeTimeout(10, TimeUnit.SECONDS) + .build() + // TODO: Paste your Nvidia API Key here + private val API_KEY = "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" + private val API_URL = "https://integrate.api.nvidia.com/v1/chat/completions" + + fun generateResponse(image: Bitmap, prompt: String, onResponse: (String?) -> Unit) { + val base64Image = encodeImageToBase64(image) + + val json = JSONObject() + json.put("model", "meta/llama-3.2-11b-vision-instruct") + json.put("max_tokens", 512) + json.put("temperature", 0.7) + json.put("top_p", 0.7) + + val messagesArray = JSONArray() + + val userMessage = JSONObject() + userMessage.put("role", "user") + + val contentArray = JSONArray() + + val textContent = JSONObject() + textContent.put("type", "text") + textContent.put("text", "You are an assistant for a blind person. Give a concise, helpful 1-2 sentence answer. $prompt") + + val imageContent = JSONObject() + imageContent.put("type", "image_url") + val imageUrlObj = JSONObject() + imageUrlObj.put("url", "data:image/jpeg;base64,$base64Image") + imageContent.put("image_url", imageUrlObj) + + contentArray.put(textContent) + contentArray.put(imageContent) + + userMessage.put("content", contentArray) + messagesArray.put(userMessage) + + json.put("messages", messagesArray) + + val requestBody = json.toString().toRequestBody("application/json".toMediaType()) + + val request = Request.Builder() + .url(API_URL) + .post(requestBody) + .addHeader("Authorization", "Bearer $API_KEY") + .addHeader("Accept", "application/json") + .build() + + client.newCall(request).enqueue(object : Callback { + override fun onFailure(call: Call, e: IOException) { + Log.e("NvidiaApiClient", "API request failed", e) + onResponse(null) + } + + override fun onResponse(call: Call, response: Response) { + val responseBody = response.body?.string() + if (response.isSuccessful && responseBody != null) { + try { + val jsonObject = JSONObject(responseBody) + val answer = jsonObject.getJSONArray("choices") + .getJSONObject(0) + .getJSONObject("message") + .getString("content") + onResponse(answer) + } catch (e: Exception) { + Log.e("NvidiaApiClient", "Error parsing Nvidia response", e) + onResponse(null) + } + } else { + Log.e("NvidiaApiClient", "Nvidia API error: ${response.code} $responseBody") + onResponse(null) + } + } + }) + } + + private fun encodeImageToBase64(bitmap: Bitmap): String { + // Resize image to prevent massive payload + val maxWidth = 800 + val ratio = maxWidth.toFloat() / bitmap.width + val height = (bitmap.height * ratio).toInt() + val resized = Bitmap.createScaledBitmap(bitmap, maxWidth, height, true) + + val outputStream = ByteArrayOutputStream() + resized.compress(Bitmap.CompressFormat.JPEG, 70, outputStream) + return Base64.encodeToString(outputStream.toByteArray(), Base64.NO_WRAP) + } +} diff --git a/app/src/main/java/com/example/smartblindstick/OcrProcessor.kt b/app/src/main/java/com/example/smartblindstick/OcrProcessor.kt new file mode 100644 index 0000000..d019910 --- /dev/null +++ b/app/src/main/java/com/example/smartblindstick/OcrProcessor.kt @@ -0,0 +1,57 @@ +package com.example.smartblindstick + +import android.content.Context +import android.util.Log +import androidx.camera.core.ImageProxy +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.text.TextRecognition +import com.google.mlkit.vision.text.latin.TextRecognizerOptions + +class OcrProcessor(private val context: Context) { + + private val recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS) + + // Cooldown mechanism to avoid reading the same text repeatedly + private var isProcessing = false + private var lastSpokenText = "" + private var lastSpokenTime: Long = 0 + + @androidx.annotation.OptIn(androidx.camera.core.ExperimentalGetImage::class) + fun processImageProxy(imageProxy: ImageProxy, onTextFound: (String) -> Unit) { + if (isProcessing) { + imageProxy.close() + return + } + + val mediaImage = imageProxy.image + if (mediaImage != null) { + isProcessing = true + val image = InputImage.fromMediaImage(mediaImage, imageProxy.imageInfo.rotationDegrees) + + recognizer.process(image) + .addOnSuccessListener { visionText -> + val detectedText = visionText.text + if (detectedText.isNotBlank()) { + Log.d("OcrProcessor", "Found text: $detectedText") + + val currentTime = System.currentTimeMillis() + // Avoid repeating the exact same text within 10 seconds + if (detectedText != lastSpokenText || (currentTime - lastSpokenTime > 10000)) { + lastSpokenText = detectedText + lastSpokenTime = currentTime + onTextFound(detectedText) + } + } + } + .addOnFailureListener { e -> + Log.e("OcrProcessor", "Text recognition failed", e) + } + .addOnCompleteListener { + isProcessing = false + imageProxy.close() + } + } else { + imageProxy.close() + } + } +} diff --git a/app/src/main/java/com/example/smartblindstick/SpatialAudioManager.kt b/app/src/main/java/com/example/smartblindstick/SpatialAudioManager.kt new file mode 100644 index 0000000..9d21fc1 --- /dev/null +++ b/app/src/main/java/com/example/smartblindstick/SpatialAudioManager.kt @@ -0,0 +1,60 @@ +package com.example.smartblindstick + +import android.util.Log + +class SpatialAudioManager(private val onSpeak: (String, Float, Float, Float) -> Unit) { + + private var isEnabled = false + private var lastSpokenTime = 0L + private var lastSpokenTarget = "" + + fun setEnabled(enabled: Boolean) { + isEnabled = enabled + Log.d("SpatialAudioManager", "Spatial audio enabled: $enabled") + } + + fun isEnabled(): Boolean = isEnabled + + fun processDetections(detections: List, imageWidth: Int, imageHeight: Int) { + if (!isEnabled || detections.isEmpty()) return + + val currentTime = System.currentTimeMillis() + val prominentDetection = detections.minByOrNull { it.distanceMeters } ?: return + + // Only tell when object is close + if (prominentDetection.distanceMeters > 1.0f) return + + val position = getPositionDescription(prominentDetection.centerX, prominentDetection.centerY, imageWidth, imageHeight) + + // 3D Audio processing + val pan = (prominentDetection.centerX / imageWidth) * 2f - 1f + // Volume decreases as distance increases (0.5m = 1.0, 1.0m = 0.5) + val volume = (1.0f - (prominentDetection.distanceMeters - 0.5f).coerceAtLeast(0f)).coerceIn(0.2f, 1.0f) + // Pitch changes based on elevation (top of screen = higher pitch, bottom = lower pitch) + val normalizedY = prominentDetection.centerY / imageHeight + val pitch = 1.2f - (0.4f * normalizedY) // ranges from ~1.2 (top) to ~0.8 (bottom) + + val announcement = "${prominentDetection.label} detected, $position" + + // Prevent repeating the same target constantly + if (announcement != lastSpokenTarget || currentTime - lastSpokenTime > 10000) { + onSpeak(announcement, pan, volume, pitch) + lastSpokenTarget = announcement + lastSpokenTime = currentTime + } + } + + private fun getPositionDescription(centerX: Float, centerY: Float, imageWidth: Int, imageHeight: Int): String { + val hPos = when { + centerX < imageWidth / 3 -> "left" + centerX > 2 * imageWidth / 3 -> "right" + else -> "center" + } + val vPos = when { + centerY < imageHeight / 3 -> "top" + centerY > 2 * imageHeight / 3 -> "bottom" + else -> "middle" + } + return if (hPos == "center" && vPos == "middle") "center" else "$vPos $hPos" + } +} diff --git a/app/src/main/java/com/example/smartblindstick/VisionProcessor.kt b/app/src/main/java/com/example/smartblindstick/VisionProcessor.kt new file mode 100644 index 0000000..3b21f7e --- /dev/null +++ b/app/src/main/java/com/example/smartblindstick/VisionProcessor.kt @@ -0,0 +1,316 @@ +package com.example.smartblindstick + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.graphics.Matrix +import android.util.Log +import androidx.camera.core.ImageProxy +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.face.FaceDetection +import com.google.mlkit.vision.face.FaceDetectorOptions + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import org.tensorflow.lite.Interpreter +import java.io.FileInputStream +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.nio.channels.FileChannel +import kotlin.math.sqrt + +class VisionProcessor(private val context: Context) { + + // Options for Face Detection (ML Kit) + private val faceDetectorOptions = FaceDetectorOptions.Builder() + .setPerformanceMode(FaceDetectorOptions.PERFORMANCE_MODE_FAST) + .build() + private val faceDetector = FaceDetection.getClient(faceDetectorOptions) + + + + // Face Recognition (MobileFaceNet) + private var faceInterpreter: Interpreter? = null + private val knownFaces = mutableMapOf() + private val FACE_IMAGE_SIZE = 112 + private val EMBEDDING_SIZE = 192 // Standard for many MobileFaceNet models + + // Cooldown state for TTS + private var lastFaceSpeakTime: Long = 0 + + private val yoloDetector = YoloV8Detector(context) + private val nvidiaApiClient = NvidiaApiClient() + + // Managers + var spatialAudioManager: SpatialAudioManager? = null + var visualMemoryManager: VisualMemoryManager? = null + var microNavigationManager: MicroNavigationManager? = null + var autoFlashlightManager: AutoFlashlightManager? = null + + var onDetectionsUpdated: ((List, Int, Int) -> Unit)? = null + + private var isYoloProcessing = false + private val scope = CoroutineScope(Dispatchers.Default) + + init { + initializeFaceInterpreter() + CoroutineScope(Dispatchers.IO).launch { + loadKnownFaces() + } + } + + private fun initializeFaceInterpreter() { + try { + val assetFileDescriptor = context.assets.openFd("mobilefacenet.tflite") + val fileInputStream = FileInputStream(assetFileDescriptor.fileDescriptor) + val fileChannel = fileInputStream.channel + val startOffset = assetFileDescriptor.startOffset + val declaredLength = assetFileDescriptor.declaredLength + val mappedByteBuffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength) + + faceInterpreter = Interpreter(mappedByteBuffer) + Log.d("VisionProcessor", "MobileFaceNet loaded successfully.") + } catch (e: Exception) { + Log.e("VisionProcessor", "Failed to load MobileFaceNet. Make sure mobilefacenet.tflite is in assets/", e) + } + } + + private fun loadKnownFaces() { + processAssetDir("known_faces", null) + } + + private fun processAssetDir(dirPath: String, personNameOverride: String?) { + try { + val fileNames = context.assets.list(dirPath) ?: return + + for (fileName in fileNames) { + if (fileName == "README.txt") continue + + val assetPath = "$dirPath/$fileName" + val subFiles = context.assets.list(assetPath) + + if (!subFiles.isNullOrEmpty()) { + // It's a directory, use its name as the person's name + processAssetDir(assetPath, fileName) + } else { + // It's a file + val inputStream = try { + context.assets.open(assetPath) + } catch (e: Exception) { + null + } + if (inputStream == null) continue + + val bitmap = BitmapFactory.decodeStream(inputStream) ?: continue + + val inputImage = InputImage.fromBitmap(bitmap, 0) + faceDetector.process(inputImage).addOnSuccessListener { faces -> + var faceBitmap = bitmap + if (faces.isNotEmpty()) { + val face = faces[0] + val boundingBox = face.boundingBox + + val x = maxOf(0, boundingBox.left) + val y = maxOf(0, boundingBox.top) + val width = minOf(bitmap.width - x, boundingBox.width()) + val height = minOf(bitmap.height - y, boundingBox.height()) + + if (width > 0 && height > 0) { + faceBitmap = Bitmap.createBitmap(bitmap, x, y, width, height) + } + } else { + Log.w("VisionProcessor", "ML Kit didn't find a face in ${fileName}, using whole image as fallback") + } + + val embedding = getFaceEmbedding(faceBitmap) + if (embedding != null) { + val name = personNameOverride ?: fileName.substringBeforeLast(".") + knownFaces[name] = embedding + Log.d("VisionProcessor", "Registered known face: $name") + } + } + } + } + } catch (e: Exception) { + Log.e("VisionProcessor", "Error loading known faces from $dirPath", e) + } + } + + @androidx.annotation.OptIn(androidx.camera.core.ExperimentalGetImage::class) + fun processImageProxy(imageProxy: ImageProxy) { + val mediaImage = imageProxy.image + if (mediaImage != null) { + val image = InputImage.fromMediaImage(mediaImage, imageProxy.imageInfo.rotationDegrees) + + // Fix: Rotate the bitmap so the ML Kit bounding box aligns correctly! + val rawBitmap = imageProxy.toBitmap() + val luminance = calculateAverageLuminance(rawBitmap) + autoFlashlightManager?.processLuminance(luminance) + + val matrix = Matrix().apply { postRotate(imageProxy.imageInfo.rotationDegrees.toFloat()) } + val bitmap = Bitmap.createBitmap(rawBitmap, 0, 0, rawBitmap.width, rawBitmap.height, matrix, true) + + processFaces(image, bitmap) + + if (!isYoloProcessing) { + isYoloProcessing = true + scope.launch { + try { + // Run YOLOv8 detection in background + val detections = yoloDetector.detect(bitmap) + + // Update managers on main thread + withContext(Dispatchers.Main) { + spatialAudioManager?.processDetections(detections, bitmap.width, bitmap.height) + visualMemoryManager?.logSightings(detections, bitmap.width) + microNavigationManager?.processDetections(detections, bitmap.width, bitmap.height) + onDetectionsUpdated?.invoke(detections, bitmap.width, bitmap.height) + } + } catch (e: Exception) { + Log.e("VisionProcessor", "Async YOLO failed", e) + } finally { + isYoloProcessing = false + } + } + } + } + + imageProxy.close() + } + + @androidx.annotation.OptIn(androidx.camera.core.ExperimentalGetImage::class) + fun processForNvidia(imageProxy: ImageProxy, prompt: String, onResponse: (String) -> Unit) { + val rawBitmap = imageProxy.toBitmap() + val matrix = Matrix().apply { postRotate(imageProxy.imageInfo.rotationDegrees.toFloat()) } + val bitmap = Bitmap.createBitmap(rawBitmap, 0, 0, rawBitmap.width, rawBitmap.height, matrix, true) + + nvidiaApiClient.generateResponse(bitmap, prompt) { answer -> + if (answer != null) { + onResponse(answer) + } else { + onResponse("Sorry, I couldn't reach the Nvidia server.") + } + } + imageProxy.close() + } + + private fun processFaces(image: InputImage, bitmap: Bitmap) { + faceDetector.process(image) + .addOnSuccessListener { faces -> + for (face in faces) { + val bounds = face.boundingBox + + // Crop the face + val x = maxOf(0, bounds.left) + val y = maxOf(0, bounds.top) + val width = minOf(bitmap.width - x, bounds.width()) + val height = minOf(bitmap.height - y, bounds.height()) + + if (width > 0 && height > 0) { + val faceBitmap = Bitmap.createBitmap(bitmap, x, y, width, height) + val embedding = getFaceEmbedding(faceBitmap) + + if (embedding != null) { + var bestMatch = "Unknown person" + var minDistance = Float.MAX_VALUE + + // Compare with known faces + for ((name, knownEmbedding) in knownFaces) { + val distance = calculateCosineDistance(embedding, knownEmbedding) + if (distance < minDistance) { + minDistance = distance + bestMatch = name + } + } + + // Threshold for face match (adjustable, MobileFaceNet usually ~ 0.5-0.7, relaxed to 0.85 for leniency) + if (minDistance > 0.85f) { + bestMatch = "Unknown person" + } + + val currentTime = System.currentTimeMillis() + if (currentTime - lastFaceSpeakTime > 5000) { + if (context is MainActivity && bestMatch != "Unknown person") { + val pan = (x + width / 2.0f) / bitmap.width * 2f - 1f + val hPos = when { + pan < -0.33f -> "left" + pan > 0.33f -> "right" + else -> "center" + } + context.speak("$bestMatch is to your $hPos", pan) + } + lastFaceSpeakTime = currentTime + } + } + } + } + } + .addOnFailureListener { e -> + Log.e("VisionProcessor", "Face detection failed", e) + } + } + + private fun getFaceEmbedding(bitmap: Bitmap): FloatArray? { + if (faceInterpreter == null) return null + + val resizedBitmap = Bitmap.createScaledBitmap(bitmap, FACE_IMAGE_SIZE, FACE_IMAGE_SIZE, true) + val byteBuffer = ByteBuffer.allocateDirect(4 * FACE_IMAGE_SIZE * FACE_IMAGE_SIZE * 3) + byteBuffer.order(ByteOrder.nativeOrder()) + + val intValues = IntArray(FACE_IMAGE_SIZE * FACE_IMAGE_SIZE) + resizedBitmap.getPixels(intValues, 0, resizedBitmap.width, 0, 0, resizedBitmap.width, resizedBitmap.height) + + // Preprocess image: normalize to [-1, 1] usually for MobileFaceNet + var pixel = 0 + for (i in 0 until FACE_IMAGE_SIZE) { + for (j in 0 until FACE_IMAGE_SIZE) { + val value = intValues[pixel++] + byteBuffer.putFloat(((value shr 16 and 0xFF) - 127.5f) / 128f) + byteBuffer.putFloat(((value shr 8 and 0xFF) - 127.5f) / 128f) + byteBuffer.putFloat(((value and 0xFF) - 127.5f) / 128f) + } + } + + val embeddings = Array(1) { FloatArray(EMBEDDING_SIZE) } + try { + faceInterpreter?.run(byteBuffer, embeddings) + return embeddings[0] + } catch (e: Exception) { + Log.e("VisionProcessor", "Failed to extract embedding", e) + } + return null + } + + private fun calculateCosineDistance(emb1: FloatArray, emb2: FloatArray): Float { + var dotProduct = 0.0 + var norm1 = 0.0 + var norm2 = 0.0 + for (i in emb1.indices) { + dotProduct += emb1[i] * emb2[i] + norm1 += emb1[i] * emb1[i] + norm2 += emb2[i] * emb2[i] + } + return (1.0 - (dotProduct / (sqrt(norm1) * sqrt(norm2)))).toFloat() + } + + private fun calculateAverageLuminance(bitmap: Bitmap): Float { + var sum = 0f + var count = 0 + // Sample every 10th pixel to save CPU + val step = 10 + for (x in 0 until bitmap.width step step) { + for (y in 0 until bitmap.height step step) { + val pixel = bitmap.getPixel(x, y) + val r = (pixel shr 16) and 0xFF + val g = (pixel shr 8) and 0xFF + val b = pixel and 0xFF + val luma = 0.299f * r + 0.587f * g + 0.114f * b + sum += luma + count++ + } + } + return if (count > 0) sum / count else 255f + } +} diff --git a/app/src/main/java/com/example/smartblindstick/VisualMemoryManager.kt b/app/src/main/java/com/example/smartblindstick/VisualMemoryManager.kt new file mode 100644 index 0000000..d54d0cf --- /dev/null +++ b/app/src/main/java/com/example/smartblindstick/VisualMemoryManager.kt @@ -0,0 +1,207 @@ +package com.example.smartblindstick + +import android.content.Context +import android.util.Log +import com.example.smartblindstick.data.AppDatabase +import com.example.smartblindstick.data.SightingLog +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext + +class VisualMemoryManager(private val context: Context, private val onAnswerReady: (String, Float, Float, Float) -> Unit) { + var compassManager: CompassManager? = null + + private val db = AppDatabase.getDatabase(context) + private val dao = db.sightingLogDao() + private val metaApiClient = MetaApiClient() + + private val scope = CoroutineScope(Dispatchers.IO) + private var lastLogTime = 0L + + // Items we care to log (don't log floors, walls, people which move constantly) + private val objectsOfInterest = setOf( + "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", + "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", + "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", + "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", + "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", + "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", + "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", + "couch", "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", + "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", + "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush", "keys", "wallet" + ) + + fun logSightings(detections: List, imageWidth: Int) { + val currentTime = System.currentTimeMillis() + if (currentTime - lastLogTime < 3000) return // Log at most every 3 seconds + + val interestingDetections = detections.filter { objectsOfInterest.contains(it.label.lowercase()) } + if (interestingDetections.isEmpty()) return + + val currentAzimuth = compassManager?.currentAzimuth ?: 0f + + scope.launch { + try { + // Delete logs older than 24 hours + val cutoff = currentTime - (24 * 60 * 60 * 1000) + dao.deleteOlderThan(cutoff) + + val sceneContext = detections.map { it.label }.joinToString(", ") + + for (detection in interestingDetections) { + val pan = (detection.centerX / imageWidth) * 2f - 1f // -1 to 1 + val offsetAngle = pan * 30f // assuming ~60 degree horizontal FOV + val globalAzimuth = (currentAzimuth + offsetAngle + 360) % 360 + + val log = SightingLog( + objectLabel = detection.label, + timestamp = currentTime, + globalAzimuth = globalAzimuth, + confidence = detection.confidence, + sceneContext = sceneContext + ) + dao.insert(log) + Log.d("VisualMemoryManager", "Logged: ${detection.label} at globalAzimuth $globalAzimuth") + } + lastLogTime = currentTime + } catch (e: Exception) { + Log.e("VisualMemoryManager", "Error logging sightings", e) + } + } + } + + fun answerQuery(query: String) { + // Simple NLP to extract the object from "where is my [object]" + val words = query.lowercase().replace("?", "").split(" ") + var targetObject = "" + for (obj in objectsOfInterest) { + if (words.contains(obj) || query.lowercase().contains(obj)) { + targetObject = obj + break + } + } + + if (targetObject.isEmpty()) { + onAnswerReady("I'm not sure what object you are looking for.", 0f, 1.0f, 1.0f) + return + } + + scope.launch { + val logs = dao.searchLogs(targetObject) + if (logs.isEmpty()) { + withContext(Dispatchers.Main) { + onAnswerReady("I haven't seen your $targetObject recently.", 0f, 1.0f, 1.0f) + } + return@launch + } + + // Build prompt + val contextBuilder = StringBuilder() + val now = System.currentTimeMillis() + val currentAzimuth = compassManager?.currentAzimuth ?: 0f + + logs.forEach { log -> + val minutesAgo = (now - log.timestamp) / (60 * 1000) + val timeStr = if (minutesAgo == 0L) "just now" else "$minutesAgo minutes ago" + + var diff = (log.globalAzimuth - currentAzimuth + 360) % 360 + if (diff > 180) diff -= 360 + + val relativeDirection = when { + diff in -30f..30f -> "directly in front of you" + diff in 30f..150f -> "to your right" + diff in -150f..-30f -> "to your left" + else -> "behind you" + } + + contextBuilder.append("- Seen $timeStr $relativeDirection. Other objects nearby: ${log.sceneContext}\n") + } + + val prompt = """ + You are a helpful assistant for a blind person. The user asked: "$query" + Based on the following recent sightings of the object from their camera, give a concise and natural 1-2 sentence answer. + Sightings: + $contextBuilder + """.trimIndent() + + val answer = metaApiClient.generateResponse(prompt) + withContext(Dispatchers.Main) { + // If it was seen recently, let's use the latest log for panning + val latest = logs.first() + var diff = (latest.globalAzimuth - currentAzimuth + 360) % 360 + if (diff > 180) diff -= 360 + + val relativeDirection = when { + diff in -30f..30f -> "directly in front of you" + diff in 30f..150f -> "to your right" + diff in -150f..-30f -> "to your left" + else -> "behind you" + } + + // Calculate pan continuously from -1 to 1 based on sine of angle + val diffRadians = Math.toRadians(diff.toDouble()) + val pan = Math.sin(diffRadians).toFloat() + + // Volume: 1.0 when in front (0 deg), 0.5 when behind (180 deg) + val absDiff = Math.abs(diff) + val volume = (1.0f - 0.5f * (absDiff / 180f)).coerceIn(0.5f, 1.0f) + + // Pitch: slightly lower if behind to simulate acoustic shadowing + val pitch = if (absDiff > 90f) 0.9f else 1.0f + + if (answer != null) { + onAnswerReady(answer, pan, volume, pitch) + } else { + // Fallback if API fails + val minutesAgo = (now - latest.timestamp) / (60 * 1000) + val timeStr = if (minutesAgo == 0L) "just now" else "$minutesAgo minutes ago" + onAnswerReady("I saw your $targetObject $timeStr $relativeDirection.", pan, volume, pitch) + } + } + } + } + + fun exportMetadata(onCsvReady: ((String) -> Unit)? = null) { + scope.launch { + try { + val logs = dao.getAllLogs() + if (logs.isEmpty()) { + withContext(Dispatchers.Main) { + onAnswerReady("No metadata available to export.", 0f, 1.0f, 1.0f) + } + return@launch + } + + val csvHeader = "ID,Timestamp,ObjectLabel,GlobalAzimuth,Confidence,SceneContext\n" + val csvContent = java.lang.StringBuilder(csvHeader) + logs.forEach { + val safeContext = it.sceneContext.replace(",", ";") + csvContent.append("${it.id},${it.timestamp},${it.objectLabel},${it.globalAzimuth},${it.confidence},$safeContext\n") + } + + withContext(Dispatchers.Main) { + if (onCsvReady != null) { + onCsvReady(csvContent.toString()) + } else { + // Fallback to direct save if no callback provided + val downloadsDir = context.getExternalFilesDir(android.os.Environment.DIRECTORY_DOWNLOADS) + if (downloadsDir != null) { + val file = java.io.File(downloadsDir, "SmartBlindStick_Metadata.csv") + file.writeText(csvContent.toString()) + onAnswerReady("Metadata successfully exported to the Downloads folder.", 0f, 1.0f, 1.0f) + } else { + onAnswerReady("Failed to access Downloads folder.", 0f, 1.0f, 1.0f) + } + } + } + } catch (e: Exception) { + Log.e("VisualMemoryManager", "Error exporting metadata", e) + withContext(Dispatchers.Main) { + onAnswerReady("Error exporting metadata.", 0f, 1.0f, 1.0f) + } + } + } + } +} diff --git a/app/src/main/java/com/example/smartblindstick/VoiceAssistantManager.kt b/app/src/main/java/com/example/smartblindstick/VoiceAssistantManager.kt new file mode 100644 index 0000000..defd816 --- /dev/null +++ b/app/src/main/java/com/example/smartblindstick/VoiceAssistantManager.kt @@ -0,0 +1,139 @@ +package com.example.smartblindstick + +import android.content.Context +import android.content.Intent +import android.os.Bundle +import android.speech.RecognitionListener +import android.speech.RecognizerIntent +import android.speech.SpeechRecognizer +import android.util.Log + +import android.os.Handler +import android.os.Looper + +class VoiceAssistantManager( + private val context: Context, + private val onCommandReceived: (String) -> Unit +) { + private var speechRecognizer: SpeechRecognizer? = null + private var isListening = false + private var isAwake = false + private var wakeTime: Long = 0 + private val AWAKE_DURATION = 10000L // 10 seconds of listening for commands after wake word + private val mainHandler = Handler(Looper.getMainLooper()) + + init { + mainHandler.post { + initializeRecognizer() + } + } + + private fun initializeRecognizer() { + if (SpeechRecognizer.isRecognitionAvailable(context)) { + speechRecognizer = SpeechRecognizer.createSpeechRecognizer(context) + speechRecognizer?.setRecognitionListener(object : RecognitionListener { + override fun onReadyForSpeech(params: Bundle?) {} + override fun onBeginningOfSpeech() {} + override fun onRmsChanged(rmsdB: Float) {} + override fun onBufferReceived(buffer: ByteArray?) {} + override fun onEndOfSpeech() {} + + override fun onError(error: Int) { + Log.e("VoiceAssistant", "Speech recognition error: $error") + isListening = false + } + + override fun onResults(results: Bundle?) { + val matches = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION) + if (!matches.isNullOrEmpty()) { + val text = matches[0].lowercase() + Log.d("VoiceAssistant", "Heard: $text") + processText(text) + } + isListening = false + } + + override fun onPartialResults(partialResults: Bundle?) {} + override fun onEvent(eventType: Int, params: Bundle?) {} + }) + } else { + Log.e("VoiceAssistant", "Speech recognition is not available on this device.") + } + } + + fun startListening() { + mainHandler.post { + if (isListening || speechRecognizer == null) return@post + + val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { + putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM) + putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 1) + } + + try { + // Instantly wake up the assistant when triggered manually via button + isAwake = true + wakeTime = System.currentTimeMillis() + + speechRecognizer?.startListening(intent) + isListening = true + Log.d("VoiceAssistant", "Started listening...") + } catch (e: Exception) { + Log.e("VoiceAssistant", "Failed to start listening", e) + } + } + } + + fun stopListening() { + mainHandler.post { + speechRecognizer?.stopListening() + isListening = false + } + } + + fun destroy() { + mainHandler.post { + speechRecognizer?.destroy() + } + } + + private fun processText(text: String) { + val currentTime = System.currentTimeMillis() + + // Reset awake state if duration passed + if (isAwake && currentTime - wakeTime > AWAKE_DURATION) { + isAwake = false + Log.d("VoiceAssistant", "Assistant went back to sleep.") + } + + // List of wake words for "Rubix" + val wakeWords = listOf("rubix", "rubik", "rubics", "rubiks", "ruby", "rubic") + var matchedWakeWord = "" + for (word in wakeWords) { + if (text.contains(word)) { + matchedWakeWord = word + break + } + } + + if (matchedWakeWord.isNotEmpty()) { + isAwake = true + wakeTime = currentTime + + // Extract the command after the wake word + val command = text.substringAfter(matchedWakeWord).trim() + if (command.isNotEmpty()) { + onCommandReceived(command) + } else { + // Just the wake word + if (context is MainActivity) { + context.speak("hello! what can i do for u") + } + } + } else if (isAwake) { + // Already awake, process anything as a command + wakeTime = currentTime // Refresh awake timer + onCommandReceived(text) + } + } +} diff --git a/app/src/main/java/com/example/smartblindstick/YoloV8Detector.kt b/app/src/main/java/com/example/smartblindstick/YoloV8Detector.kt new file mode 100644 index 0000000..a448148 --- /dev/null +++ b/app/src/main/java/com/example/smartblindstick/YoloV8Detector.kt @@ -0,0 +1,204 @@ +package com.example.smartblindstick + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.RectF +import android.util.Log +import org.tensorflow.lite.Interpreter +import java.io.FileInputStream +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.nio.MappedByteBuffer +import java.nio.channels.FileChannel + +data class Detection( + val label: String, + val confidence: Float, + val boundingBox: RectF, + val centerX: Float, + val centerY: Float, + val distanceMeters: Float = -1f +) + +class YoloV8Detector(private val context: Context) { + private var interpreter: Interpreter? = null + private var labels = listOf() + + private val inputSize = 640 // standard yolov8 input + private val numClasses = 80 + private val numElements = 4 + numClasses // x, y, w, h, p1...p80 + private val numAnchors = 8400 // typical for yolov8n 640x640 + + // Pre-allocated buffers to prevent GC churn (saves ~15MB of garbage per frame) + private val byteBuffer = ByteBuffer.allocateDirect(4 * inputSize * inputSize * 3).apply { + order(ByteOrder.nativeOrder()) + } + private val floatArray = FloatArray(inputSize * inputSize * 3) + private val intValues = IntArray(inputSize * inputSize) + private val outputBuffer = Array(1) { Array(numElements) { FloatArray(numAnchors) } } + + init { + try { + val tfliteModel = loadModelFile("yolov8n.tflite") + val options = Interpreter.Options().apply { + numThreads = 4 + } + interpreter = Interpreter(tfliteModel, options) + labels = loadLabels("coco_labels.txt") + Log.d("YoloV8Detector", "YOLOv8 model loaded successfully.") + } catch (e: Exception) { + Log.e("YoloV8Detector", "Error loading YOLOv8 model", e) + } + } + + private fun loadModelFile(modelName: String): MappedByteBuffer { + val fileDescriptor = context.assets.openFd(modelName) + val inputStream = FileInputStream(fileDescriptor.fileDescriptor) + val fileChannel = inputStream.channel + val startOffset = fileDescriptor.startOffset + val declaredLength = fileDescriptor.declaredLength + return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength) + } + + private fun loadLabels(labelPath: String): List { + return context.assets.open(labelPath).bufferedReader().readLines() + } + + fun detect(bitmap: Bitmap): List { + if (interpreter == null) return emptyList() + + // 1. Preprocess + val resizedBitmap = Bitmap.createScaledBitmap(bitmap, inputSize, inputSize, false) + val buffer = convertBitmapToByteBuffer(resizedBitmap) + + // 2. Run inference + try { + interpreter?.run(buffer, outputBuffer) + } catch (e: Exception) { + Log.e("YoloV8Detector", "Inference failed", e) + return emptyList() + } + + // 3. Postprocess + val detections = mutableListOf() + val output = outputBuffer[0] // [84, 8400] + + for (i in 0 until numAnchors) { + var maxClassScore = 0f + var classIndex = -1 + + for (c in 0 until numClasses) { + val score = output[4 + c][i] + if (score > maxClassScore) { + maxClassScore = score + classIndex = c + } + } + + if (maxClassScore > 0.3f) { // confidence threshold + val cx = output[0][i] + val cy = output[1][i] + val w = output[2][i] + val h = output[3][i] + + // Handle coordinates which may be absolute (0-640) or already normalized (0-1) + val nx = if (cx > 2f) cx / inputSize else cx + val ny = if (cy > 2f) cy / inputSize else cy + val nw = if (w > 2f) w / inputSize else w + val nh = if (h > 2f) h / inputSize else h + + // Calculate bounding box in normalized coordinates + val left = nx - nw / 2 + val top = ny - nh / 2 + val right = nx + nw / 2 + val bottom = ny + nh / 2 + + // Map back to original image dimensions + val rectF = RectF( + left * bitmap.width, + top * bitmap.height, + right * bitmap.width, + bottom * bitmap.height + ) + + // Heuristic distance calculation based on normalized area + val areaRatio = nw * nh + val distance = 0.5f / maxOf(0.01f, Math.sqrt(areaRatio.toDouble()).toFloat()) + + val label = if (classIndex in labels.indices) labels[classIndex] else "Unknown" + + detections.add( + Detection( + label = label, + confidence = maxClassScore, + boundingBox = rectF, + centerX = nx * bitmap.width, + centerY = ny * bitmap.height, + distanceMeters = distance + ) + ) + } + } + + return applyNMS(detections) + } + + private fun convertBitmapToByteBuffer(bitmap: Bitmap): ByteBuffer { + byteBuffer.rewind() + bitmap.getPixels(intValues, 0, bitmap.width, 0, 0, bitmap.width, bitmap.height) + + var pixel = 0 + var floatIndex = 0 + val numPixels = inputSize * inputSize + while (pixel < numPixels) { + val value = intValues[pixel++] + // Normalize to 0.0 - 1.0 for YOLOv8 + floatArray[floatIndex++] = ((value shr 16) and 0xFF) / 255.0f + floatArray[floatIndex++] = ((value shr 8) and 0xFF) / 255.0f + floatArray[floatIndex++] = (value and 0xFF) / 255.0f + } + + // Bulk put using FloatBuffer view is astronomically faster than byteBuffer.putFloat in a loop + byteBuffer.asFloatBuffer().put(floatArray) + return byteBuffer + } + + private fun applyNMS(detections: List, iouThreshold: Float = 0.45f): List { + val sortedDetections = detections.sortedByDescending { it.confidence } + val finalDetections = mutableListOf() + + for (detection in sortedDetections) { + var shouldAdd = true + for (finalDetection in finalDetections) { + if (detection.label == finalDetection.label) { + val iou = calculateIoU(detection.boundingBox, finalDetection.boundingBox) + if (iou > iouThreshold) { + shouldAdd = false + break + } + } + } + if (shouldAdd) { + finalDetections.add(detection) + } + } + return finalDetections + } + + private fun calculateIoU(box1: RectF, box2: RectF): Float { + val intersectionLeft = maxOf(box1.left, box2.left) + val intersectionTop = maxOf(box1.top, box2.top) + val intersectionRight = minOf(box1.right, box2.right) + val intersectionBottom = minOf(box1.bottom, box2.bottom) + + if (intersectionRight < intersectionLeft || intersectionBottom < intersectionTop) { + return 0.0f + } + + val intersectionArea = (intersectionRight - intersectionLeft) * (intersectionBottom - intersectionTop) + val box1Area = box1.width() * box1.height() + val box2Area = box2.width() * box2.height() + + return intersectionArea / (box1Area + box2Area - intersectionArea) + } +} diff --git a/app/src/main/java/com/example/smartblindstick/data/AppDatabase.kt b/app/src/main/java/com/example/smartblindstick/data/AppDatabase.kt new file mode 100644 index 0000000..a9a6a2b --- /dev/null +++ b/app/src/main/java/com/example/smartblindstick/data/AppDatabase.kt @@ -0,0 +1,30 @@ +package com.example.smartblindstick.data + +import android.content.Context +import androidx.room.Database +import androidx.room.Room +import androidx.room.RoomDatabase + +@Database(entities = [SightingLog::class], version = 2, exportSchema = false) +abstract class AppDatabase : RoomDatabase() { + abstract fun sightingLogDao(): SightingLogDao + + companion object { + @Volatile + private var INSTANCE: AppDatabase? = null + + fun getDatabase(context: Context): AppDatabase { + return INSTANCE ?: synchronized(this) { + val instance = Room.databaseBuilder( + context.applicationContext, + AppDatabase::class.java, + "smart_blind_stick_database" + ) + .fallbackToDestructiveMigration() + .build() + INSTANCE = instance + instance + } + } + } +} diff --git a/app/src/main/java/com/example/smartblindstick/data/SightingLog.kt b/app/src/main/java/com/example/smartblindstick/data/SightingLog.kt new file mode 100644 index 0000000..9084a08 --- /dev/null +++ b/app/src/main/java/com/example/smartblindstick/data/SightingLog.kt @@ -0,0 +1,14 @@ +package com.example.smartblindstick.data + +import androidx.room.Entity +import androidx.room.PrimaryKey + +@Entity(tableName = "sighting_logs") +data class SightingLog( + @PrimaryKey(autoGenerate = true) val id: Int = 0, + val objectLabel: String, + val timestamp: Long, + val globalAzimuth: Float, + val confidence: Float, + val sceneContext: String +) diff --git a/app/src/main/java/com/example/smartblindstick/data/SightingLogDao.kt b/app/src/main/java/com/example/smartblindstick/data/SightingLogDao.kt new file mode 100644 index 0000000..b72c809 --- /dev/null +++ b/app/src/main/java/com/example/smartblindstick/data/SightingLogDao.kt @@ -0,0 +1,19 @@ +package com.example.smartblindstick.data + +import androidx.room.Dao +import androidx.room.Insert +import androidx.room.Query + +@Dao +interface SightingLogDao { + @Insert + fun insert(log: SightingLog) + + @Query("SELECT * FROM sighting_logs WHERE objectLabel LIKE '%' || :query || '%' ORDER BY timestamp DESC LIMIT 5") + fun searchLogs(query: String): List + @Query("SELECT * FROM sighting_logs ORDER BY timestamp DESC") + fun getAllLogs(): List + + @Query("DELETE FROM sighting_logs WHERE timestamp < :cutoffTime") + fun deleteOlderThan(cutoffTime: Long) +} diff --git a/app/src/main/java/com/example/smartblindstick/ui/theme/Color.kt b/app/src/main/java/com/example/smartblindstick/ui/theme/Color.kt new file mode 100644 index 0000000..fc2c7c1 --- /dev/null +++ b/app/src/main/java/com/example/smartblindstick/ui/theme/Color.kt @@ -0,0 +1,11 @@ +package com.example.smartblindstick.ui.theme + +import androidx.compose.ui.graphics.Color + +val Purple80 = Color(0xFFD0BCFF) +val PurpleGrey80 = Color(0xFFCCC2DC) +val Pink80 = Color(0xFFEFB8C8) + +val Purple40 = Color(0xFF6650a4) +val PurpleGrey40 = Color(0xFF625b71) +val Pink40 = Color(0xFF7D5260) \ No newline at end of file diff --git a/app/src/main/java/com/example/smartblindstick/ui/theme/Theme.kt b/app/src/main/java/com/example/smartblindstick/ui/theme/Theme.kt new file mode 100644 index 0000000..0a43f0e --- /dev/null +++ b/app/src/main/java/com/example/smartblindstick/ui/theme/Theme.kt @@ -0,0 +1,58 @@ +package com.example.smartblindstick.ui.theme + +import android.app.Activity +import android.os.Build +import androidx.compose.foundation.isSystemInDarkTheme +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.darkColorScheme +import androidx.compose.material3.dynamicDarkColorScheme +import androidx.compose.material3.dynamicLightColorScheme +import androidx.compose.material3.lightColorScheme +import androidx.compose.runtime.Composable +import androidx.compose.ui.platform.LocalContext + +private val DarkColorScheme = darkColorScheme( + primary = Purple80, + secondary = PurpleGrey80, + tertiary = Pink80 +) + +private val LightColorScheme = lightColorScheme( + primary = Purple40, + secondary = PurpleGrey40, + tertiary = Pink40 + + /* Other default colors to override + background = Color(0xFFFFFBFE), + surface = Color(0xFFFFFBFE), + onPrimary = Color.White, + onSecondary = Color.White, + onTertiary = Color.White, + onBackground = Color(0xFF1C1B1F), + onSurface = Color(0xFF1C1B1F), + */ +) + +@Composable +fun SmartBlindStickTheme( + darkTheme: Boolean = isSystemInDarkTheme(), + // Dynamic color is available on Android 12+ + dynamicColor: Boolean = true, + content: @Composable () -> Unit +) { + val colorScheme = when { + dynamicColor && Build.VERSION.SDK_INT >= Build.VERSION_CODES.S -> { + val context = LocalContext.current + if (darkTheme) dynamicDarkColorScheme(context) else dynamicLightColorScheme(context) + } + + darkTheme -> DarkColorScheme + else -> LightColorScheme + } + + MaterialTheme( + colorScheme = colorScheme, + typography = Typography, + content = content + ) +} \ No newline at end of file diff --git a/app/src/main/java/com/example/smartblindstick/ui/theme/Type.kt b/app/src/main/java/com/example/smartblindstick/ui/theme/Type.kt new file mode 100644 index 0000000..7241297 --- /dev/null +++ b/app/src/main/java/com/example/smartblindstick/ui/theme/Type.kt @@ -0,0 +1,34 @@ +package com.example.smartblindstick.ui.theme + +import androidx.compose.material3.Typography +import androidx.compose.ui.text.TextStyle +import androidx.compose.ui.text.font.FontFamily +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.sp + +// Set of Material typography styles to start with +val Typography = Typography( + bodyLarge = TextStyle( + fontFamily = FontFamily.Default, + fontWeight = FontWeight.Normal, + fontSize = 16.sp, + lineHeight = 24.sp, + letterSpacing = 0.5.sp + ) + /* Other default text styles to override + titleLarge = TextStyle( + fontFamily = FontFamily.Default, + fontWeight = FontWeight.Normal, + fontSize = 22.sp, + lineHeight = 28.sp, + letterSpacing = 0.sp + ), + labelSmall = TextStyle( + fontFamily = FontFamily.Default, + fontWeight = FontWeight.Medium, + fontSize = 11.sp, + lineHeight = 16.sp, + letterSpacing = 0.5.sp + ) + */ +) \ No newline at end of file diff --git a/app/src/main/keepRules/rules.keep b/app/src/main/keepRules/rules.keep new file mode 100644 index 0000000..d7e081a --- /dev/null +++ b/app/src/main/keepRules/rules.keep @@ -0,0 +1,12 @@ +# Add project specific R8 rules here. +# AGP will combine all keep rule files in src/main/keepRules to pass to R8 +# +# For more details, see +# https://d.android.com/r/tools/r8/keep-rules + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} \ No newline at end of file diff --git a/app/src/main/res/drawable/ic_launcher_background.xml b/app/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 0000000..07d5da9 --- /dev/null +++ b/app/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/app/src/main/res/drawable/ic_launcher_foreground.xml b/app/src/main/res/drawable/ic_launcher_foreground.xml new file mode 100644 index 0000000..2b068d1 --- /dev/null +++ b/app/src/main/res/drawable/ic_launcher_foreground.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml b/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml new file mode 100644 index 0000000..6f3b755 --- /dev/null +++ b/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml b/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml new file mode 100644 index 0000000..6f3b755 --- /dev/null +++ b/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/app/src/main/res/mipmap-hdpi/ic_launcher.webp b/app/src/main/res/mipmap-hdpi/ic_launcher.webp new file mode 100644 index 0000000..c209e78 Binary files /dev/null and b/app/src/main/res/mipmap-hdpi/ic_launcher.webp differ diff --git a/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp b/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp new file mode 100644 index 0000000..b2dfe3d Binary files /dev/null and b/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp differ diff --git a/app/src/main/res/mipmap-mdpi/ic_launcher.webp b/app/src/main/res/mipmap-mdpi/ic_launcher.webp new file mode 100644 index 0000000..4f0f1d6 Binary files /dev/null and b/app/src/main/res/mipmap-mdpi/ic_launcher.webp differ diff --git a/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp b/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp new file mode 100644 index 0000000..62b611d Binary files /dev/null and b/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp differ diff --git a/app/src/main/res/mipmap-xhdpi/ic_launcher.webp b/app/src/main/res/mipmap-xhdpi/ic_launcher.webp new file mode 100644 index 0000000..948a307 Binary files /dev/null and b/app/src/main/res/mipmap-xhdpi/ic_launcher.webp differ diff --git a/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp b/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp new file mode 100644 index 0000000..1b9a695 Binary files /dev/null and b/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp differ diff --git a/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp b/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp new file mode 100644 index 0000000..28d4b77 Binary files /dev/null and b/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp differ diff --git a/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp b/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp new file mode 100644 index 0000000..9287f50 Binary files /dev/null and b/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp differ diff --git a/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp b/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp new file mode 100644 index 0000000..aa7d642 Binary files /dev/null and b/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp differ diff --git a/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp b/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp new file mode 100644 index 0000000..9126ae3 Binary files /dev/null and b/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp differ diff --git a/app/src/main/res/values/colors.xml b/app/src/main/res/values/colors.xml new file mode 100644 index 0000000..f8c6127 --- /dev/null +++ b/app/src/main/res/values/colors.xml @@ -0,0 +1,10 @@ + + + #FFBB86FC + #FF6200EE + #FF3700B3 + #FF03DAC5 + #FF018786 + #FF000000 + #FFFFFFFF + \ No newline at end of file diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml new file mode 100644 index 0000000..72d98a9 --- /dev/null +++ b/app/src/main/res/values/strings.xml @@ -0,0 +1,3 @@ + + smart blind stick + \ No newline at end of file diff --git a/app/src/main/res/values/themes.xml b/app/src/main/res/values/themes.xml new file mode 100644 index 0000000..1328afc --- /dev/null +++ b/app/src/main/res/values/themes.xml @@ -0,0 +1,5 @@ + + + +