diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ceafa1bc..9386546b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -57,7 +57,8 @@ barcodeScanningCommon = "17.0.0" playServicesMlkitBarcodeScanning = "18.3.1" protobuf = "0.10.0" firebaseCrashlyticsBuildtools = "3.0.7" -uwb = "1.0.0" +uwb = "1.1.0-alpha03" +arcore = "1.48.0" telecom = "1.1.0-SNAPSHOT" @@ -66,6 +67,7 @@ androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = androidx-fragment-compose = { module = "androidx.fragment:fragment-compose", version.ref = "fragmentCompose" } androidx-navigation-compose = { module = "androidx.navigation:navigation-compose", version.ref = "navigationCompose" } androidx-uwb = { module = "androidx.core.uwb:uwb", version.ref = "uwb" } +arcore = { module = "com.google.ar:core", version.ref = "arcore" } androidx-photopicker-compose = { module = "androidx.photopicker:photopicker-compose", version.ref = "photopickerCompose" } junit = { group = "junit", name = "junit", version.ref = "junit" } androidx-junit = { group = "androidx.test.ext", name = "junit", version.ref = "junitVersion" } diff --git a/samples/connectivity/UwbRanging/build.gradle.kts b/samples/connectivity/UwbRanging/build.gradle.kts index 1b7f7d20..8a515424 100644 --- a/samples/connectivity/UwbRanging/build.gradle.kts +++ b/samples/connectivity/UwbRanging/build.gradle.kts @@ -72,6 +72,7 @@ protobuf { dependencies { implementation(libs.androidx.uwb) + implementation(libs.arcore) implementation(project(mapOf("path" to ":samples:connectivity:audio"))) implementation(libs.androidx.activity.compose) diff --git a/samples/connectivity/UwbRanging/src/main/AndroidManifest.xml b/samples/connectivity/UwbRanging/src/main/AndroidManifest.xml index d9cd6d4f..0df776bb 100644 --- a/samples/connectivity/UwbRanging/src/main/AndroidManifest.xml +++ b/samples/connectivity/UwbRanging/src/main/AndroidManifest.xml @@ -23,13 +23,18 @@ + + + + settings.toBuilder().setSensorFusionEnabled(enabled).build() + } + } + } + companion object { private const val STORE_FILE_NAME = "app_settings.pb" diff --git a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/data/UwbRangingControlSource.kt b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/data/UwbRangingControlSource.kt index f6e55640..22e97b58 100644 --- a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/data/UwbRangingControlSource.kt +++ b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/data/UwbRangingControlSource.kt @@ -31,6 +31,8 @@ interface UwbRangingControlSource { var configType: ConfigType + var sensorFusionEnabled: Boolean + fun updateEndpointId(id: String) fun start() diff --git a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/data/UwbRangingControlSourceImpl.kt b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/data/UwbRangingControlSourceImpl.kt index bfb46c1c..efe4ee28 100644 --- a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/data/UwbRangingControlSourceImpl.kt +++ b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/data/UwbRangingControlSourceImpl.kt @@ -43,7 +43,11 @@ internal class UwbRangingControlSourceImpl( private var uwbEndpoint = UwbEndpoint(endpointId, SecureRandom.getSeed(8)) private var uwbSessionScope: UwbSessionScope = - getSessionScope(DeviceType.CONTROLLER, ConfigType.CONFIG_UNICAST_DS_TWR) + getSessionScope( + DeviceType.CONTROLLER, + ConfigType.CONFIG_UNICAST_DS_TWR, + sensorFusionEnabled = false, + ) private var rangingJob: Job? = null @@ -57,16 +61,21 @@ internal class UwbRangingControlSourceImpl( override val isRunning = runningStateFlow.asStateFlow() - private fun getSessionScope(deviceType: DeviceType, configType: ConfigType): UwbSessionScope { + private fun getSessionScope( + deviceType: DeviceType, + configType: ConfigType, + sensorFusionEnabled: Boolean, + ): UwbSessionScope { return when (deviceType) { - DeviceType.CONTROLEE -> uwbConnectionManager.controleeUwbScope(uwbEndpoint) + DeviceType.CONTROLEE -> + uwbConnectionManager.controleeUwbScope(uwbEndpoint, sensorFusionEnabled) DeviceType.CONTROLLER -> uwbConnectionManager.controllerUwbScope(uwbEndpoint, when (configType) { ConfigType.CONFIG_UNICAST_DS_TWR -> RangingParameters.CONFIG_UNICAST_DS_TWR ConfigType.CONFIG_MULTICAST_DS_TWR -> RangingParameters.CONFIG_MULTICAST_DS_TWR ConfigType.CONFIG_PROVISIONED_UNICAST -> RangingParameters.CONFIG_PROVISIONED_UNICAST_DS_TWR else -> throw java.lang.IllegalStateException() - }) + }, sensorFusionEnabled) else -> throw IllegalStateException() } } @@ -79,7 +88,7 @@ internal class UwbRangingControlSourceImpl( Delegates.observable(DeviceType.CONTROLLER) { _, oldValue, newValue -> if (oldValue != newValue) { stop() - uwbSessionScope = getSessionScope(newValue, configType) + uwbSessionScope = getSessionScope(newValue, configType, sensorFusionEnabled) } } @@ -87,7 +96,15 @@ internal class UwbRangingControlSourceImpl( Delegates.observable(ConfigType.CONFIG_UNICAST_DS_TWR) { _, oldValue, newValue -> if (oldValue != newValue) { stop() - uwbSessionScope = getSessionScope(deviceType, newValue) + uwbSessionScope = getSessionScope(deviceType, newValue, sensorFusionEnabled) + } + } + + override var sensorFusionEnabled: Boolean by + Delegates.observable(false) { _, oldValue, newValue -> + if (oldValue != newValue) { + stop() + uwbSessionScope = getSessionScope(deviceType, configType, newValue) } } @@ -95,7 +112,7 @@ internal class UwbRangingControlSourceImpl( if (id != uwbEndpoint.id) { stop() uwbEndpoint = UwbEndpoint(id, SecureRandom.getSeed(8)) - uwbSessionScope = getSessionScope(deviceType, configType) + uwbSessionScope = getSessionScope(deviceType, configType, sensorFusionEnabled) } } diff --git a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/UwbRangingActivity.kt b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/UwbRangingActivity.kt index 539b1a7a..a762909d 100644 --- a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/UwbRangingActivity.kt +++ b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/UwbRangingActivity.kt @@ -27,12 +27,16 @@ import android.widget.Toast import androidx.activity.ComponentActivity import androidx.activity.compose.setContent import com.google.uwb.hellouwb.HelloUwbApplication +import com.google.ar.core.ArCoreApk +import com.google.ar.core.exceptions.UnavailableUserDeclinedInstallationException private const val PERMISSION_REQUEST_CODE = 1234 class UwbRangingActivity : ComponentActivity() { + private var userRequestedInstall = true + override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) requestPermissions() @@ -57,31 +61,63 @@ class UwbRangingActivity : ComponentActivity() { } } - private fun requestPermissions() { - if (!arePermissionsGranted()) { - requestPermissions(PERMISSIONS_REQUIRED, PERMISSION_REQUEST_CODE) + override fun onResume() { + super.onResume() + if ( + checkCallingOrSelfPermission(Manifest.permission.CAMERA) == + PackageManager.PERMISSION_GRANTED + ) { + ArCoreApk.getInstance().checkAvailabilityAsync(this) { availability -> + if (availability.isSupported) { + try { + when (ArCoreApk.getInstance().requestInstall(this, userRequestedInstall)) { + ArCoreApk.InstallStatus.INSTALLED -> {} + ArCoreApk.InstallStatus.INSTALL_REQUESTED -> { + userRequestedInstall = false + } + } + } catch (e: UnavailableUserDeclinedInstallationException) { + Log.e("UWB Sample", "User declined ARCore installation", e) + } catch (e: Exception) { + Log.e("UWB Sample", "ARCore installation check failed", e) + } + } + } } } - private fun arePermissionsGranted(): Boolean { - for (permission in PERMISSIONS_REQUIRED) { - if (checkCallingOrSelfPermission(permission) != PackageManager.PERMISSION_GRANTED) { - return false - } + private fun requestPermissions() { + val missingPermissions = getMissingPermissions() + if (missingPermissions.isNotEmpty()) { + requestPermissions(missingPermissions, PERMISSION_REQUEST_CODE) } - return true } + private fun getMissingPermissions(): Array = + PERMISSIONS_REQUIRED.filter { + checkCallingOrSelfPermission(it) != PackageManager.PERMISSION_GRANTED + }.toTypedArray() + override fun onRequestPermissionsResult( requestCode: Int, permissions: Array, grantResults: IntArray, ) { super.onRequestPermissionsResult(requestCode, permissions, grantResults) - for (result in grantResults) { - if (result != PackageManager.PERMISSION_GRANTED) { - requestPermissions() + if (requestCode != PERMISSION_REQUEST_CODE) return + // Don't re-request here: denied (or non-requestable) permissions are returned immediately, + // which would cause an endless request loop. + val deniedPermissions = + permissions.filterIndexed { index, _ -> + grantResults.getOrNull(index) != PackageManager.PERMISSION_GRANTED } + if (deniedPermissions.isNotEmpty()) { + Log.w("UWB Sample", "Permissions denied: $deniedPermissions") + Toast.makeText( + applicationContext, + "UWB ranging requires all requested permissions", + Toast.LENGTH_LONG, + ).show() } } @@ -90,8 +126,6 @@ class UwbRangingActivity : ComponentActivity() { private val PERMISSIONS_REQUIRED_BEFORE_T = listOf( // Permissions needed by Nearby Connection - Manifest.permission.BLUETOOTH, - Manifest.permission.BLUETOOTH_ADMIN, Manifest.permission.BLUETOOTH_SCAN, Manifest.permission.BLUETOOTH_ADVERTISE, Manifest.permission.BLUETOOTH_CONNECT, @@ -100,7 +134,10 @@ class UwbRangingActivity : ComponentActivity() { Manifest.permission.CHANGE_WIFI_STATE, // permission required by UWB API - Manifest.permission.UWB_RANGING + Manifest.permission.UWB_RANGING, + + // permission required by ARCore for UWB sensor fusion + Manifest.permission.CAMERA, ) private val PERMISSIONS_REQUIRED_T = @@ -108,12 +145,21 @@ class UwbRangingActivity : ComponentActivity() { Manifest.permission.NEARBY_WIFI_DEVICES, ) + // Runtime permission required by Nearby Connections starting in Android 17 + private val PERMISSIONS_REQUIRED_CINNAMON_BUN = + arrayOf( + Manifest.permission.ACCESS_LOCAL_NETWORK, + ) + private val PERMISSIONS_REQUIRED = PERMISSIONS_REQUIRED_BEFORE_T.toMutableList() .apply { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { addAll(PERMISSIONS_REQUIRED_T) } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.CINNAMON_BUN) { + addAll(PERMISSIONS_REQUIRED_CINNAMON_BUN) + } } .toTypedArray() } diff --git a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/control/ControlViewModel.kt b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/control/ControlViewModel.kt index 5c73f592..156cfc9f 100644 --- a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/control/ControlViewModel.kt +++ b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/control/ControlViewModel.kt @@ -51,16 +51,20 @@ class ControlViewModel( launch { uwbRangingControlSource .observeRangingResults() - .filterIsInstance() + .mapNotNull { + when (it) { + is EndpointEvents.PositionUpdated -> it.position.distance + is EndpointEvents.SensorFusionEstimateUpdated -> it.estimate.distance + else -> null + } + } .collect { - it.position.distance?.let { - val state = _uiState.value as ControlUiState.LockState - if (!state.isLocked && it.value > LOCK_DISTANCE) { - _uiState.update { ControlUiState.LockState(isLocked = true) } - } - if (state.isLocked && it.value < UNLOCK_DISTANCE) { - _uiState.update { ControlUiState.LockState(isLocked = false) } - } + val state = _uiState.value as ControlUiState.LockState + if (!state.isLocked && it.value > LOCK_DISTANCE) { + _uiState.update { ControlUiState.LockState(isLocked = true) } + } + if (state.isLocked && it.value < UNLOCK_DISTANCE) { + _uiState.update { ControlUiState.LockState(isLocked = false) } } } } diff --git a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/home/HomeScreen.kt b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/home/HomeScreen.kt index 8418edb9..2d57db1b 100644 --- a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/home/HomeScreen.kt +++ b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/home/HomeScreen.kt @@ -64,17 +64,49 @@ fun HomeScreen(uiState: HomeUiState, modifier: Modifier = Modifier) { modifier = modifier ) { innerPadding -> Column(modifier = Modifier.padding(innerPadding)) { - Row(modifier = Modifier.padding(innerPadding)) { + Row { ConnectStatusBar( uiState.connectedEndpoints.map { it.endpoint }, uiState.disconnectedEndpoints ) } + SensorFusionStatusSection(uiState) Row { RangingPlot(uiState.connectedEndpoints) } } } } +@Composable +fun SensorFusionStatusSection(uiState: HomeUiState) { + Column(modifier = Modifier.padding(horizontal = 16.dp)) { + if (uiState.isSensorFusionFallback) { + val reasonSuffix = uiState.fallbackReason?.let { " ($it)" } ?: "" + Text( + text = "Fell back to standard ranging$reasonSuffix", + color = Color.Red, + ) + } + uiState.connectedEndpoints.forEach { connectedEndpoint -> + val endpointName = connectedEndpoint.endpoint.id.split("|")[0] + when (val estimateState = connectedEndpoint.estimateState) { + is EstimateState.Drifting -> { + Text( + text = "$endpointName: DRIFTING", + color = Color.Red, + ) + } + is EstimateState.Imprecise -> { + Text( + text = "$endpointName: IMPRECISE: (${estimateState.reason})", + color = Color.Red, + ) + } + is EstimateState.Precise -> {} + } + } + } +} + @OptIn(ExperimentalMaterial3Api::class) @Composable fun HomeTopAppBar( @@ -108,10 +140,15 @@ fun RangingPlot(connectedEndpoints: List) { val scale = drawPolar(center) connectedEndpoints.forEachIndexed { index, endpoint -> endpoint.position.distance?.let { distance -> - endpoint.position.azimuth?.let { azimuth -> + val azimuth = + when (endpoint.estimateState) { + is EstimateState.Imprecise -> 0.0f + else -> endpoint.position.azimuth?.value + } + azimuth?.let { drawPosition( distance.value, - azimuth.value, + it, scale = scale, centerOffset = center, color = ENDPOINT_COLORS[index % ENDPOINT_COLORS.size] diff --git a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/home/HomeViewModel.kt b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/home/HomeViewModel.kt index 9353198b..5c4692f0 100644 --- a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/home/HomeViewModel.kt +++ b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/home/HomeViewModel.kt @@ -18,13 +18,19 @@ package com.google.uwb.hellouwb.ui.home +import androidx.core.uwb.RangingMeasurement import androidx.core.uwb.RangingPosition +import androidx.core.uwb.SensorFusionResult import androidx.lifecycle.ViewModel import androidx.lifecycle.ViewModelProvider import androidx.lifecycle.viewModelScope import com.google.uwb.hellouwb.data.UwbRangingControlSource import com.google.uwb.uwbranging.EndpointEvents import com.google.uwb.uwbranging.UwbEndpoint +import kotlin.math.asin +import kotlin.math.atan2 +import kotlin.math.cos +import kotlin.math.sin import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.* @@ -32,21 +38,32 @@ import kotlinx.coroutines.flow.* class HomeViewModel(uwbRangingControlSource: UwbRangingControlSource) : ViewModel() { private val _uiState: MutableStateFlow = - MutableStateFlow(HomeUiStateImpl(listOf(), listOf(), false)) + MutableStateFlow(HomeUiStateImpl(listOf(), listOf(), false, false, null)) private val endpoints = mutableListOf() private val endpointPositions = mutableMapOf() + private val endpointEstimateStates = mutableMapOf() private var isRanging = false + private var isSensorFusionFallback = false + private var fallbackReason: String? = null private fun updateUiState(): HomeUiState { return HomeUiStateImpl( endpoints .mapNotNull { endpoint -> - endpointPositions[endpoint]?.let { position -> ConnectedEndpoint(endpoint, position) } + endpointPositions[endpoint]?.let { position -> + ConnectedEndpoint( + endpoint, + position, + endpointEstimateStates[endpoint] ?: EstimateState.Precise, + ) + } } .toList(), endpoints.filter { !endpointPositions.containsKey(it) }.toList(), - isRanging + isRanging, + isSensorFusionFallback, + fallbackReason, ) } @@ -58,12 +75,41 @@ class HomeViewModel(uwbRangingControlSource: UwbRangingControlSource) : ViewMode .onEach { result -> when (result) { is EndpointEvents.EndpointFound -> endpoints.add(result.endpoint) - is EndpointEvents.UwbDisconnected -> endpointPositions.remove(result.endpoint) - is EndpointEvents.PositionUpdated -> endpointPositions[result.endpoint] = result.position + is EndpointEvents.UwbDisconnected -> { + endpointPositions.remove(result.endpoint) + endpointEstimateStates.remove(result.endpoint) + } + is EndpointEvents.PositionUpdated -> { + endpointPositions[result.endpoint] = result.position + endpointEstimateStates.remove(result.endpoint) + } + is EndpointEvents.SensorFusionEstimateUpdated -> { + isSensorFusionFallback = false + fallbackReason = null + when (val estimate = result.estimate) { + is SensorFusionResult.PreciseEstimate -> { + endpointPositions[result.endpoint] = estimateToBoresightPosition(estimate) + endpointEstimateStates[result.endpoint] = EstimateState.Precise + } + is SensorFusionResult.DriftingEstimate -> { + endpointPositions[result.endpoint] = estimateToBoresightPosition(estimate) + endpointEstimateStates[result.endpoint] = EstimateState.Drifting + } + is SensorFusionResult.ImpreciseEstimate -> { + endpointPositions[result.endpoint] = estimateToBoresightPosition(estimate) + endpointEstimateStates[result.endpoint] = + EstimateState.Imprecise(estimateFailureReasonToString(estimate.reason)) + } + } + } + is EndpointEvents.SensorFusionFallback -> { + isSensorFusionFallback = true + fallbackReason = fallbackReasonToString(result.fallback.reason) + } is EndpointEvents.EndpointLost -> { endpoints.remove(result.endpoint) - endpointPositions.remove(result.endpoint) + endpointEstimateStates.remove(result.endpoint) } else -> return@onEach } @@ -77,6 +123,9 @@ class HomeViewModel(uwbRangingControlSource: UwbRangingControlSource) : ViewMode if (!running) { endpoints.clear() endpointPositions.clear() + endpointEstimateStates.clear() + isSensorFusionFallback = false + fallbackReason = null } _uiState.update { updateUiState() } } @@ -84,12 +133,48 @@ class HomeViewModel(uwbRangingControlSource: UwbRangingControlSource) : ViewMode } private data class HomeUiStateImpl( - override val connectedEndpoints: List, - override val disconnectedEndpoints: List, - override val isRanging: Boolean, + override val connectedEndpoints: List, + override val disconnectedEndpoints: List, + override val isRanging: Boolean, + override val isSensorFusionFallback: Boolean, + override val fallbackReason: String?, ) : HomeUiState companion object { + internal fun estimateToBoresightPosition(estimate: SensorFusionResult.Estimate): RangingPosition { + if (estimate is SensorFusionResult.ImpreciseEstimate) { + return RangingPosition( + estimate.distance, + RangingMeasurement(0.0f), + null, + estimate.elapsedRealtimeNanos, + ) + } + val azimuth = estimate.azimuth + val elevation = estimate.elevation + if (azimuth != null && elevation != null) { + val azRad = Math.toRadians(azimuth.value.toDouble()) + val elRad = Math.toRadians(elevation.value.toDouble()) + val x = cos(elRad) * sin(azRad) + val y = cos(elRad) * cos(azRad) + val z = sin(elRad) + val boresightAzimuthDeg = Math.toDegrees(atan2(x, -z)).toFloat() + val boresightElevationDeg = Math.toDegrees(asin(y.coerceIn(-1.0, 1.0))).toFloat() + return RangingPosition( + estimate.distance, + RangingMeasurement(boresightAzimuthDeg), + RangingMeasurement(boresightElevationDeg), + estimate.elapsedRealtimeNanos, + ) + } + return RangingPosition( + estimate.distance, + azimuth, + elevation, + estimate.elapsedRealtimeNanos, + ) + } + fun provideFactory( uwbRangingControlSource: UwbRangingControlSource ): ViewModelProvider.Factory = @@ -99,13 +184,67 @@ class HomeViewModel(uwbRangingControlSource: UwbRangingControlSource) : ViewMode return HomeViewModel(uwbRangingControlSource) as T } } + + internal fun estimateFailureReasonToString(reason: Int): String = + when (reason) { + SensorFusionResult.ESTIMATE_FAILURE_REASON_NOT_AVAILABLE -> + "ESTIMATE_FAILURE_REASON_NOT_AVAILABLE" + SensorFusionResult.ESTIMATE_FAILURE_REASON_INSUFFICIENT_LIGHT -> + "ESTIMATE_FAILURE_REASON_INSUFFICIENT_LIGHT" + SensorFusionResult.ESTIMATE_FAILURE_REASON_EXCESSIVE_MOTION -> + "ESTIMATE_FAILURE_REASON_EXCESSIVE_MOTION" + SensorFusionResult.ESTIMATE_FAILURE_REASON_INSUFFICIENT_FEATURES -> + "ESTIMATE_FAILURE_REASON_INSUFFICIENT_FEATURES" + SensorFusionResult.ESTIMATE_FAILURE_REASON_CAMERA_UNAVAILABLE -> + "ESTIMATE_FAILURE_REASON_CAMERA_UNAVAILABLE" + SensorFusionResult.ESTIMATE_FAILURE_REASON_PEER_MOVING -> + "ESTIMATE_FAILURE_REASON_PEER_MOVING" + SensorFusionResult.ESTIMATE_FAILURE_REASON_INCONCLUSIVE_RESULT -> + "ESTIMATE_FAILURE_REASON_INCONCLUSIVE_RESULT" + SensorFusionResult.ESTIMATE_FAILURE_REASON_BAD_STATE -> + "ESTIMATE_FAILURE_REASON_BAD_STATE" + else -> "UNKNOWN ($reason)" + } + + internal fun fallbackReasonToString(reason: Int): String = + when (reason) { + SensorFusionResult.FALLBACK_REASON_NOT_AVAILABLE -> + "FALLBACK_REASON_NOT_AVAILABLE" + SensorFusionResult.FALLBACK_REASON_HARDWARE_AOA_PRECEDENCE -> + "FALLBACK_REASON_HARDWARE_AOA_PRECEDENCE" + SensorFusionResult.FALLBACK_REASON_ARCORE_CAMERA_NOT_AVAILABLE -> + "FALLBACK_REASON_ARCORE_CAMERA_NOT_AVAILABLE" + SensorFusionResult.FALLBACK_REASON_ARCORE_SDK_TOO_OLD -> + "FALLBACK_REASON_ARCORE_SDK_TOO_OLD" + SensorFusionResult.FALLBACK_REASON_ARCORE_DEVICE_NOT_COMPATIBLE -> + "FALLBACK_REASON_ARCORE_DEVICE_NOT_COMPATIBLE" + SensorFusionResult.FALLBACK_REASON_ARCORE_MISSING_GL_CONTEXT -> + "FALLBACK_REASON_ARCORE_MISSING_GL_CONTEXT" + else -> "UNKNOWN ($reason)" + } } } +sealed interface EstimateState { + object Precise : EstimateState + + object Drifting : EstimateState + + data class Imprecise(val reason: String) : EstimateState +} + interface HomeUiState { val connectedEndpoints: List val disconnectedEndpoints: List val isRanging: Boolean + val isSensorFusionFallback: Boolean + get() = false + val fallbackReason: String? + get() = null } -data class ConnectedEndpoint(val endpoint: UwbEndpoint, val position: RangingPosition) +data class ConnectedEndpoint( + val endpoint: UwbEndpoint, + val position: RangingPosition, + val estimateState: EstimateState = EstimateState.Precise, +) diff --git a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/send/SendViewModel.kt b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/send/SendViewModel.kt index 31d573d3..41189447 100644 --- a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/send/SendViewModel.kt +++ b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/send/SendViewModel.kt @@ -21,9 +21,11 @@ package com.google.uwb.hellouwb.ui.send import android.content.ContentResolver import android.net.Uri import androidx.core.net.toUri +import androidx.core.uwb.SensorFusionResult import androidx.lifecycle.ViewModel import androidx.lifecycle.ViewModelProvider import com.google.uwb.hellouwb.data.UwbRangingControlSource +import com.google.uwb.hellouwb.ui.home.HomeViewModel import com.google.uwb.uwbranging.EndpointEvents import com.google.uwb.uwbranging.UwbEndpoint import kotlinx.coroutines.CoroutineScope @@ -102,10 +104,20 @@ class SendViewModel( uwbRangingControlSource .observeRangingResults() .filterNot { it.endpoint in endpointsSent } - .filterIsInstance() .collect { event -> - event.position.azimuth?.let { azimuth -> - if (azimuth.value > -5.0f && azimuth.value < 5.0f) { + val azimuth = + when (event) { + is EndpointEvents.PositionUpdated -> event.position.azimuth + is EndpointEvents.SensorFusionEstimateUpdated -> + if (event.estimate is SensorFusionResult.ImpreciseEstimate) { + null + } else { + HomeViewModel.estimateToBoresightPosition(event.estimate).azimuth + } + else -> null + } + azimuth?.let { + if (it.value > -5.0f && it.value < 5.0f) { endpointsSent.add(event.endpoint) uwbRangingControlSource.sendOobMessage(event.endpoint, bytesToSend) val endpointDisplayName = event.endpoint.id.split("|")[0] diff --git a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/settings/SettingsRoute.kt b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/settings/SettingsRoute.kt index 8f7897cb..1b8423a3 100644 --- a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/settings/SettingsRoute.kt +++ b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/settings/SettingsRoute.kt @@ -29,6 +29,7 @@ fun SettingsRoute(settingsViewModel: SettingsViewModel) { uiState = uiState, updateDeviceDisplayName = { settingsViewModel.updateDeviceDisplayName(it) }, updateDeviceType = { settingsViewModel.updateDeviceType(it) }, - updateConfigType = { settingsViewModel.updateConfigType(it) } + updateConfigType = { settingsViewModel.updateConfigType(it) }, + updateSensorFusionEnabled = { settingsViewModel.updateSensorFusionEnabled(it) }, ) } diff --git a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/settings/SettingsScreen.kt b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/settings/SettingsScreen.kt index ebefb73b..d876de6b 100644 --- a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/settings/SettingsScreen.kt +++ b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/settings/SettingsScreen.kt @@ -41,6 +41,7 @@ fun SettingsScreen( updateDeviceDisplayName: (String) -> Unit, updateDeviceType: (DeviceType) -> Unit, updateConfigType: (ConfigType) -> Unit, + updateSensorFusionEnabled: (Boolean) -> Unit, modifier: Modifier = Modifier, ) { Scaffold( @@ -151,6 +152,18 @@ fun SettingsScreen( } } } + + Row(verticalAlignment = Alignment.CenterVertically) { + Text("Sensor Fusion", Modifier.padding(end = 12.dp)) + var sensorFusionEnabled by remember { mutableStateOf(uiState.sensorFusionEnabled) } + Switch( + checked = sensorFusionEnabled, + onCheckedChange = { + updateSensorFusionEnabled(it) + sensorFusionEnabled = it + }, + ) + } } } } @@ -163,9 +176,11 @@ fun PreviewSettingsScreen() { .setDeviceDisplayName("UWB") .setDeviceType(DeviceType.CONTROLEE) .setConfigType(ConfigType.CONFIG_PROVISIONED_UNICAST) + .setSensorFusionEnabled(true) .build(), {}, {}, + {}, {} ) } diff --git a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/settings/SettingsViewModel.kt b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/settings/SettingsViewModel.kt index cd7e14f0..3525176a 100644 --- a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/settings/SettingsViewModel.kt +++ b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/hellouwb/ui/settings/SettingsViewModel.kt @@ -55,6 +55,14 @@ class SettingsViewModel( settingsStore.updateConfigType(configType) } + fun updateSensorFusionEnabled(enabled: Boolean) { + if (enabled == uiState.value.sensorFusionEnabled) { + return + } + uwbRangingControlSource.sensorFusionEnabled = enabled + settingsStore.updateSensorFusionEnabled(enabled) + } + companion object { fun provideFactory( uwbRangingControlSource: UwbRangingControlSource, diff --git a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/uwbranging/EndpointEvents.kt b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/uwbranging/EndpointEvents.kt index 6fcb8fe5..ed52a575 100644 --- a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/uwbranging/EndpointEvents.kt +++ b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/uwbranging/EndpointEvents.kt @@ -19,6 +19,7 @@ package com.google.uwb.uwbranging import androidx.core.uwb.RangingPosition +import androidx.core.uwb.SensorFusionResult /** A data class for ranging result update. */ abstract class EndpointEvents internal constructor() { @@ -33,6 +34,26 @@ abstract class EndpointEvents internal constructor() { data class PositionUpdated(override val endpoint: UwbEndpoint, val position: RangingPosition) : EndpointEvents() + /** + * Sensor fusion estimate update. + * + * @property estimate Sensor fusion estimate of the UWB device during Ranging + */ + data class SensorFusionEstimateUpdated( + override val endpoint: UwbEndpoint, + val estimate: SensorFusionResult.Estimate, + ) : EndpointEvents() + + /** + * Sensor fusion fallback update. + * + * @property fallback Sensor fusion fallback result when falling back to standard ranging + */ + data class SensorFusionFallback( + override val endpoint: UwbEndpoint, + val fallback: SensorFusionResult.SensorFusionFallback, + ) : EndpointEvents() + /** A ranging result with peer disconnected status update. */ data class UwbDisconnected(override val endpoint: UwbEndpoint) : EndpointEvents() diff --git a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/uwbranging/UwbConnectionManager.kt b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/uwbranging/UwbConnectionManager.kt index 46301698..b9dfa83f 100644 --- a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/uwbranging/UwbConnectionManager.kt +++ b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/uwbranging/UwbConnectionManager.kt @@ -26,9 +26,16 @@ import com.google.uwb.uwbranging.impl.UwbConnectionManagerImpl * starts the UWB ranging. */ interface UwbConnectionManager { - fun controllerUwbScope(endpoint: UwbEndpoint, configId: Int): UwbSessionScope + fun controllerUwbScope( + endpoint: UwbEndpoint, + configId: Int, + sensorFusionEnabled: Boolean = false, + ): UwbSessionScope - fun controleeUwbScope(endpoint: UwbEndpoint): UwbSessionScope + fun controleeUwbScope( + endpoint: UwbEndpoint, + sensorFusionEnabled: Boolean = false, + ): UwbSessionScope companion object { fun getInstance(context: Context): UwbConnectionManager { diff --git a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/uwbranging/impl/UwbConnectionManagerImpl.kt b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/uwbranging/impl/UwbConnectionManagerImpl.kt index 6913a86f..117595da 100644 --- a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/uwbranging/impl/UwbConnectionManagerImpl.kt +++ b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/uwbranging/impl/UwbConnectionManagerImpl.kt @@ -33,19 +33,26 @@ internal class UwbConnectionManagerImpl( private val uwbManager = UwbManager.createInstance(context) - override fun controllerUwbScope(endpoint: UwbEndpoint, configId: Int): UwbSessionScope { + override fun controllerUwbScope( + endpoint: UwbEndpoint, + configId: Int, + sensorFusionEnabled: Boolean, + ): UwbSessionScope { val connector = NearbyControllerConnector(endpoint, configId, NearbyConnections(context, dispatcher)) { uwbManager.controllerSessionScope() } - return UwbSessionScopeImpl(endpoint, connector) + return UwbSessionScopeImpl(endpoint, connector, sensorFusionEnabled) } - override fun controleeUwbScope(endpoint: UwbEndpoint): UwbSessionScope { + override fun controleeUwbScope( + endpoint: UwbEndpoint, + sensorFusionEnabled: Boolean, + ): UwbSessionScope { val connector = NearbyControleeConnector(endpoint, NearbyConnections(context, dispatcher)) { uwbManager.controleeSessionScope() } - return UwbSessionScopeImpl(endpoint, connector) + return UwbSessionScopeImpl(endpoint, connector, sensorFusionEnabled) } } diff --git a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/uwbranging/impl/UwbSessionScopeImpl.kt b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/uwbranging/impl/UwbSessionScopeImpl.kt index abd479b6..72b4b6ab 100644 --- a/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/uwbranging/impl/UwbSessionScopeImpl.kt +++ b/samples/connectivity/UwbRanging/src/main/java/com/google/uwb/uwbranging/impl/UwbSessionScopeImpl.kt @@ -20,6 +20,8 @@ package com.google.uwb.uwbranging.impl import androidx.core.uwb.RangingParameters import androidx.core.uwb.RangingResult +import androidx.core.uwb.SensorFusionParameters +import androidx.core.uwb.SensorFusionResult import androidx.core.uwb.UwbAddress import androidx.core.uwb.UwbDevice import com.google.uwb.uwbranging.EndpointEvents @@ -35,6 +37,7 @@ import kotlinx.coroutines.launch internal class UwbSessionScopeImpl( private val localEndpoint: UwbEndpoint, private val connector: OobConnector, + private val sensorFusionEnabled: Boolean = false, ) : UwbSessionScope { private val localAddresses = mutableSetOf() @@ -74,29 +77,39 @@ internal class UwbSessionScopeImpl( private fun ProducerScope.processEndpointFound( event: UwbOobEvent.UwbEndpointFound, - ): Flow { + ): Flow { remoteDeviceMap[event.endpointAddress] = event.endpoint localAddresses.add(event.sessionScope.localAddress) val rangingParameters = RangingParameters( - event.configId, - event.sessionId, - event.subSessionid, - event.sessionKeyInfo, - event.subSessionKeyInfo, - event.complexChannel, - listOf(UwbDevice(event.endpointAddress)), - RangingParameters.RANGING_UPDATE_RATE_FREQUENT + uwbConfigType = event.configId, + sessionId = event.sessionId, + subSessionId = event.subSessionid, + sessionKeyInfo = event.sessionKeyInfo, + subSessionKeyInfo = event.subSessionKeyInfo, + complexChannel = event.complexChannel, + peerDevices = listOf(UwbDevice(event.endpointAddress)), + updateRateType = RangingParameters.RANGING_UPDATE_RATE_FREQUENT, + // Sensor fusion provides the angle estimates, so hardware AoA is disabled. + isAoaDisabled = sensorFusionEnabled, ) trySend(EndpointEvents.EndpointFound(event.endpoint)) - return event.sessionScope.prepareSession(rangingParameters) + return if (sensorFusionEnabled) { + event.sessionScope.prepareSession(SensorFusionParameters(rangingParameters)) + } else { + event.sessionScope.prepareSession(rangingParameters) + } } - private fun ProducerScope.sendResult(result: RangingResult) { + private fun ProducerScope.sendResult(result: SensorFusionResult) { val endpoint = if (localAddresses.contains(result.device.address)) localEndpoint else remoteDeviceMap[result.device.address] ?: return when (result) { + is SensorFusionResult.Estimate -> + trySend(EndpointEvents.SensorFusionEstimateUpdated(endpoint, result)) + is SensorFusionResult.SensorFusionFallback -> + trySend(EndpointEvents.SensorFusionFallback(endpoint, result)) is RangingResult.RangingResultPosition -> trySend(EndpointEvents.PositionUpdated(endpoint, result.position)) is RangingResult.RangingResultPeerDisconnected -> diff --git a/samples/connectivity/UwbRanging/src/main/proto/appsetttings.proto b/samples/connectivity/UwbRanging/src/main/proto/appsetttings.proto index 678d803f..3041fdc7 100644 --- a/samples/connectivity/UwbRanging/src/main/proto/appsetttings.proto +++ b/samples/connectivity/UwbRanging/src/main/proto/appsetttings.proto @@ -36,4 +36,5 @@ message AppSettings { string device_display_name = 2; string device_uuid = 3; ConfigType config_type = 4; + bool sensor_fusion_enabled = 5; }