UwbRanging: Integrate sensor fusion feature

gradle/libs.versions.toml

@@ -57,7 +57,8 @@ barcodeScanningCommon = "17.0.0"
 playServicesMlkitBarcodeScanning = "18.3.1"
 protobuf = "0.10.0"
 firebaseCrashlyticsBuildtools = "3.0.7"
-uwb = "1.0.0"
+uwb = "1.1.0-alpha03"
+arcore = "1.48.0"
 telecom = "1.1.0-SNAPSHOT"
 
 
@@ -66,6 +67,7 @@ androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref =
 androidx-fragment-compose = { module = "androidx.fragment:fragment-compose", version.ref = "fragmentCompose" }
 androidx-navigation-compose = { module = "androidx.navigation:navigation-compose", version.ref = "navigationCompose" }
 androidx-uwb = { module = "androidx.core.uwb:uwb", version.ref = "uwb" }
+arcore = { module = "com.google.ar:core", version.ref = "arcore" }
 androidx-photopicker-compose = { module = "androidx.photopicker:photopicker-compose", version.ref = "photopickerCompose" }
 junit = { group = "junit", name = "junit", version.ref = "junit" }
 androidx-junit = { group = "androidx.test.ext", name = "junit", version.ref = "junitVersion" }

samples/connectivity/UwbRanging/build.gradle.kts

@@ -72,6 +72,7 @@ protobuf {
 
 dependencies {
     implementation(libs.androidx.uwb)
+    implementation(libs.arcore)
     implementation(project(mapOf("path" to ":samples:connectivity:audio")))
 
     implementation(libs.androidx.activity.compose)

samples/connectivity/UwbRanging/src/main/AndroidManifest.xml

@@ -23,13 +23,18 @@
     <uses-permission android:name="android.permission.ACCESS_WIFI_STATE" />
     <uses-permission android:name="android.permission.CHANGE_WIFI_STATE" />
     <uses-permission android:name="android.permission.NEARBY_WIFI_DEVICES" />
+    <uses-permission android:name="android.permission.ACCESS_LOCAL_NETWORK" />
     <uses-permission android:name="android.permission.UWB_RANGING" />
+    <uses-permission android:name="android.permission.CAMERA" />
 
     <application
         android:name="com.google.uwb.hellouwb.HelloUwbApplication"
         android:allowBackup="true"
         android:dataExtractionRules="@xml/data_extraction_rules"
         android:fullBackupContent="@xml/backup_rules">
+        <meta-data
+            android:name="com.google.ar.core"
+            android:value="optional" />
         <activity
             android:name="com.google.uwb.hellouwb.ui.UwbRangingActivity"
             android:exported="true"

samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/data/AppContainerImpl.kt

@@ -54,6 +54,7 @@ internal class AppContainerImpl(
                     afterLoading()
                 } else {
                     rangingResultSource.deviceType = it.deviceType
+                    rangingResultSource.sensorFusionEnabled = it.sensorFusionEnabled
           rangingResultSource.updateEndpointId(endpointId)
         }
       }

samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/data/SettingsStore.kt

@@ -30,4 +30,6 @@ interface SettingsStore {
   fun updateConfigType(configType: ConfigType)
 
   fun updateDeviceDisplayName(displayName: String)
+
+  fun updateSensorFusionEnabled(enabled: Boolean)
 }

samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/data/SettingsStoreImpl.kt

@@ -73,6 +73,14 @@ internal class SettingsStoreImpl(
     }
   }
 
+  override fun updateSensorFusionEnabled(enabled: Boolean) {
+    coroutineScope.launch {
+      context.settingsDataStore.updateData { settings ->
+        settings.toBuilder().setSensorFusionEnabled(enabled).build()
+      }
+    }
+  }
+
   companion object {
     private const val STORE_FILE_NAME = "app_settings.pb"
 

samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/data/UwbRangingControlSource.kt

@@ -31,6 +31,8 @@ interface UwbRangingControlSource {
 
     var configType: ConfigType
 
+    var sensorFusionEnabled: Boolean
+
     fun updateEndpointId(id: String)
 
     fun start()

samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/data/UwbRangingControlSourceImpl.kt

@@ -43,7 +43,11 @@ internal class UwbRangingControlSourceImpl(
   private var uwbEndpoint = UwbEndpoint(endpointId, SecureRandom.getSeed(8))
 
   private var uwbSessionScope: UwbSessionScope =
-    getSessionScope(DeviceType.CONTROLLER, ConfigType.CONFIG_UNICAST_DS_TWR)
+    getSessionScope(
+      DeviceType.CONTROLLER,
+      ConfigType.CONFIG_UNICAST_DS_TWR,
+      sensorFusionEnabled = false,
+    )
 
   private var rangingJob: Job? = null
 
@@ -57,16 +61,21 @@ internal class UwbRangingControlSourceImpl(
 
   override val isRunning = runningStateFlow.asStateFlow()
 
-  private fun getSessionScope(deviceType: DeviceType, configType: ConfigType): UwbSessionScope {
+  private fun getSessionScope(
+    deviceType: DeviceType,
+    configType: ConfigType,
+    sensorFusionEnabled: Boolean,
+  ): UwbSessionScope {
     return when (deviceType) {
-      DeviceType.CONTROLEE -> uwbConnectionManager.controleeUwbScope(uwbEndpoint)
+      DeviceType.CONTROLEE ->
+        uwbConnectionManager.controleeUwbScope(uwbEndpoint, sensorFusionEnabled)
       DeviceType.CONTROLLER ->
         uwbConnectionManager.controllerUwbScope(uwbEndpoint, when (configType) {
           ConfigType.CONFIG_UNICAST_DS_TWR -> RangingParameters.CONFIG_UNICAST_DS_TWR
           ConfigType.CONFIG_MULTICAST_DS_TWR -> RangingParameters.CONFIG_MULTICAST_DS_TWR
           ConfigType.CONFIG_PROVISIONED_UNICAST -> RangingParameters.CONFIG_PROVISIONED_UNICAST_DS_TWR
           else -> throw java.lang.IllegalStateException()
-        })
+        }, sensorFusionEnabled)
       else -> throw IllegalStateException()
     }
   }
@@ -79,23 +88,31 @@ internal class UwbRangingControlSourceImpl(
   Delegates.observable(DeviceType.CONTROLLER) { _, oldValue, newValue ->
     if (oldValue != newValue) {
       stop()
-      uwbSessionScope = getSessionScope(newValue, configType)
+      uwbSessionScope = getSessionScope(newValue, configType, sensorFusionEnabled)
     }
   }
 
   override var configType: ConfigType by
   Delegates.observable(ConfigType.CONFIG_UNICAST_DS_TWR) { _, oldValue, newValue ->
     if (oldValue != newValue) {
       stop()
-      uwbSessionScope = getSessionScope(deviceType, newValue)
+      uwbSessionScope = getSessionScope(deviceType, newValue, sensorFusionEnabled)
+    }
+  }
+
+  override var sensorFusionEnabled: Boolean by
+  Delegates.observable(false) { _, oldValue, newValue ->
+    if (oldValue != newValue) {
+      stop()
+      uwbSessionScope = getSessionScope(deviceType, configType, newValue)
     }
   }
 
   override fun updateEndpointId(id: String) {
     if (id != uwbEndpoint.id) {
       stop()
       uwbEndpoint = UwbEndpoint(id, SecureRandom.getSeed(8))
-      uwbSessionScope = getSessionScope(deviceType, configType)
+      uwbSessionScope = getSessionScope(deviceType, configType, sensorFusionEnabled)
     }
   }
 

samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/UwbRangingActivity.kt

@@ -27,12 +27,16 @@ import android.widget.Toast
 import androidx.activity.ComponentActivity
 import androidx.activity.compose.setContent
 import com.google.uwb.hellouwb.HelloUwbApplication
+import com.google.ar.core.ArCoreApk
+import com.google.ar.core.exceptions.UnavailableUserDeclinedInstallationException
 
 
 private const val PERMISSION_REQUEST_CODE = 1234
 
 class UwbRangingActivity : ComponentActivity() {
 
+  private var userRequestedInstall = true
+
   override fun onCreate(savedInstanceState: Bundle?) {
     super.onCreate(savedInstanceState)
     requestPermissions()
@@ -57,31 +61,63 @@ class UwbRangingActivity : ComponentActivity() {
     }
   }
 
-  private fun requestPermissions() {
-    if (!arePermissionsGranted()) {
-      requestPermissions(PERMISSIONS_REQUIRED, PERMISSION_REQUEST_CODE)
+  override fun onResume() {
+    super.onResume()
+    if (
+      checkCallingOrSelfPermission(Manifest.permission.CAMERA) ==
+        PackageManager.PERMISSION_GRANTED
+    ) {
+      ArCoreApk.getInstance().checkAvailabilityAsync(this) { availability ->
+        if (availability.isSupported) {
+          try {
+            when (ArCoreApk.getInstance().requestInstall(this, userRequestedInstall)) {
+              ArCoreApk.InstallStatus.INSTALLED -> {}
+              ArCoreApk.InstallStatus.INSTALL_REQUESTED -> {
+                userRequestedInstall = false
+              }
+            }
+          } catch (e: UnavailableUserDeclinedInstallationException) {
+            Log.e("UWB Sample", "User declined ARCore installation", e)
+          } catch (e: Exception) {
+            Log.e("UWB Sample", "ARCore installation check failed", e)
+          }
+        }
+      }
     }
   }
 
-  private fun arePermissionsGranted(): Boolean {
-    for (permission in PERMISSIONS_REQUIRED) {
-      if (checkCallingOrSelfPermission(permission) != PackageManager.PERMISSION_GRANTED) {
-        return false
-      }
+  private fun requestPermissions() {
+    val missingPermissions = getMissingPermissions()
+    if (missingPermissions.isNotEmpty()) {
+      requestPermissions(missingPermissions, PERMISSION_REQUEST_CODE)
     }
-    return true
   }
 
+  private fun getMissingPermissions(): Array<String> =
+    PERMISSIONS_REQUIRED.filter {
+      checkCallingOrSelfPermission(it) != PackageManager.PERMISSION_GRANTED
+    }.toTypedArray()
+
   override fun onRequestPermissionsResult(
     requestCode: Int,
     permissions: Array<String>,
     grantResults: IntArray,
   ) {
     super.onRequestPermissionsResult(requestCode, permissions, grantResults)
-    for (result in grantResults) {
-      if (result != PackageManager.PERMISSION_GRANTED) {
-        requestPermissions()
+    if (requestCode != PERMISSION_REQUEST_CODE) return
+    // Don't re-request here: denied (or non-requestable) permissions are returned immediately,
+    // which would cause an endless request loop.
+    val deniedPermissions =
+      permissions.filterIndexed { index, _ ->
+        grantResults.getOrNull(index) != PackageManager.PERMISSION_GRANTED
       }
+    if (deniedPermissions.isNotEmpty()) {
+      Log.w("UWB Sample", "Permissions denied: $deniedPermissions")
+      Toast.makeText(
+        applicationContext,
+        "UWB ranging requires all requested permissions",
+        Toast.LENGTH_LONG,
+      ).show()
     }
   }
 
@@ -90,8 +126,6 @@ class UwbRangingActivity : ComponentActivity() {
     private val PERMISSIONS_REQUIRED_BEFORE_T =
       listOf(
         // Permissions needed by Nearby Connection
-        Manifest.permission.BLUETOOTH,
-        Manifest.permission.BLUETOOTH_ADMIN,
         Manifest.permission.BLUETOOTH_SCAN,
         Manifest.permission.BLUETOOTH_ADVERTISE,
         Manifest.permission.BLUETOOTH_CONNECT,
@@ -100,20 +134,32 @@ class UwbRangingActivity : ComponentActivity() {
         Manifest.permission.CHANGE_WIFI_STATE,
 
         // permission required by UWB API
-        Manifest.permission.UWB_RANGING
+        Manifest.permission.UWB_RANGING,
+
+        // permission required by ARCore for UWB sensor fusion
+        Manifest.permission.CAMERA,
       )
 
     private val PERMISSIONS_REQUIRED_T =
       arrayOf(
         Manifest.permission.NEARBY_WIFI_DEVICES,
       )
 
+    // Runtime permission required by Nearby Connections starting in Android 17
+    private val PERMISSIONS_REQUIRED_CINNAMON_BUN =
+      arrayOf(
+        Manifest.permission.ACCESS_LOCAL_NETWORK,
+      )
+
     private val PERMISSIONS_REQUIRED =
       PERMISSIONS_REQUIRED_BEFORE_T.toMutableList()
         .apply {
           if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
             addAll(PERMISSIONS_REQUIRED_T)
           }
+          if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.CINNAMON_BUN) {
+            addAll(PERMISSIONS_REQUIRED_CINNAMON_BUN)
+          }
         }
         .toTypedArray()
   }

samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/control/ControlViewModel.kt

@@ -51,16 +51,20 @@ class ControlViewModel(
             launch {
                 uwbRangingControlSource
                     .observeRangingResults()
-                    .filterIsInstance<EndpointEvents.PositionUpdated>()
+                    .mapNotNull {
+                        when (it) {
+                            is EndpointEvents.PositionUpdated -> it.position.distance
+                            is EndpointEvents.SensorFusionEstimateUpdated -> it.estimate.distance
+                            else -> null
+                        }
+                    }
                     .collect {
-                        it.position.distance?.let {
-                            val state = _uiState.value as ControlUiState.LockState
-                            if (!state.isLocked && it.value > LOCK_DISTANCE) {
-                                _uiState.update { ControlUiState.LockState(isLocked = true) }
-                            }
-                            if (state.isLocked && it.value < UNLOCK_DISTANCE) {
-                                _uiState.update { ControlUiState.LockState(isLocked = false) }
-                            }
+                        val state = _uiState.value as ControlUiState.LockState
+                        if (!state.isLocked && it.value > LOCK_DISTANCE) {
+                            _uiState.update { ControlUiState.LockState(isLocked = true) }
+                        }
+                        if (state.isLocked && it.value < UNLOCK_DISTANCE) {
+                            _uiState.update { ControlUiState.LockState(isLocked = false) }
                         }
                     }
             }

samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/home/HomeScreen.kt

@@ -64,17 +64,49 @@ fun HomeScreen(uiState: HomeUiState, modifier: Modifier = Modifier) {
     modifier = modifier
   ) { innerPadding ->
     Column(modifier = Modifier.padding(innerPadding)) {
-      Row(modifier = Modifier.padding(innerPadding)) {
+      Row {
         ConnectStatusBar(
           uiState.connectedEndpoints.map { it.endpoint },
           uiState.disconnectedEndpoints
         )
       }
+      SensorFusionStatusSection(uiState)
       Row { RangingPlot(uiState.connectedEndpoints) }
     }
   }
 }
 
+@Composable
+fun SensorFusionStatusSection(uiState: HomeUiState) {
+  Column(modifier = Modifier.padding(horizontal = 16.dp)) {
+    if (uiState.isSensorFusionFallback) {
+      val reasonSuffix = uiState.fallbackReason?.let { " ($it)" } ?: ""
+      Text(
+        text = "Fell back to standard ranging$reasonSuffix",
+        color = Color.Red,
+      )
+    }
+    uiState.connectedEndpoints.forEach { connectedEndpoint ->
+      val endpointName = connectedEndpoint.endpoint.id.split("|")[0]
+      when (val estimateState = connectedEndpoint.estimateState) {
+        is EstimateState.Drifting -> {
+          Text(
+            text = "$endpointName: DRIFTING",
+            color = Color.Red,
+          )
+        }
+        is EstimateState.Imprecise -> {
+          Text(
+            text = "$endpointName: IMPRECISE: (${estimateState.reason})",
+            color = Color.Red,
+          )
+        }
+        is EstimateState.Precise -> {}
+      }
+    }
+  }
+}
+
 @OptIn(ExperimentalMaterial3Api::class)
 @Composable
 fun HomeTopAppBar(
@@ -108,10 +140,15 @@ fun RangingPlot(connectedEndpoints: List<ConnectedEndpoint>) {
     val scale = drawPolar(center)
     connectedEndpoints.forEachIndexed { index, endpoint ->
       endpoint.position.distance?.let { distance ->
-        endpoint.position.azimuth?.let { azimuth ->
+        val azimuth =
+          when (endpoint.estimateState) {
+            is EstimateState.Imprecise -> 0.0f
+            else -> endpoint.position.azimuth?.value
+          }
+        azimuth?.let {
           drawPosition(
             distance.value,
-            azimuth.value,
+            it,
             scale = scale,
             centerOffset = center,
             color = ENDPOINT_COLORS[index % ENDPOINT_COLORS.size]