Scale the video start bitrate hint by connection setup time

.changeset/scale-video-start-bitrate-by-connection-setup-time.md

@@ -0,0 +1,5 @@
+---
+"client-sdk-android": patch
+---
+
+Scale the `x-google-start-bitrate` hint by connection setup time: the 1 Mbps camera cap now applies to connections that set up within 1.5 s and ramps linearly down to 300 kbps at 3.5 s or slower, with screen share capped the same way once the cap is below 1 Mbps.

livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt

@@ -18,6 +18,7 @@ package io.livekit.android.room
 
 import android.javax.sdp.MediaDescription
 import android.javax.sdp.SdpFactory
+import android.os.SystemClock
 import androidx.annotation.VisibleForTesting
 import dagger.assisted.Assisted
 import dagger.assisted.AssistedFactory
@@ -63,6 +64,8 @@ import kotlin.contracts.ExperimentalContracts
 import kotlin.contracts.InvocationKind
 import kotlin.contracts.contract
 import kotlin.math.roundToLong
+import kotlin.time.Duration
+import kotlin.time.Duration.Companion.milliseconds
 
 /**
  * @suppress
@@ -99,6 +102,15 @@ constructor(
     // available through data-channel/audio-only offers and consume it only after a
     // local video m-section successfully gets the hint.
     private var hasAppliedVideoStartBitrate = false
+
+    // When the connection attempt that created this transport began, and how long it took to
+    // set up once the primary transport connected. computeTrackStartBitrate lowers the hint for
+    // slow connections; until the setup completes, the time elapsed so far stands in for it.
+    @Volatile
+    private var connectStartedAtMs: Long? = null
+
+    @Volatile
+    private var connectionSetupTime: Duration? = null
     private var isClosed = AtomicBoolean(false)
 
     private val latestOfferId = AtomicInteger(0)
@@ -217,11 +229,20 @@ constructor(
                 // consume the video start hint. When the first video offer is created, use
                 // one connection-level value across all video m-sections so libwebrtc's
                 // last-writer-wins handling cannot depend on SDP m-section order.
+                val setupCompleted = connectionSetupTime != null
+                val connectionSetupTime = connectionSetupTimeForOffer()
                 val connectionStartBitrate = if (!hasAppliedVideoStartBitrate) {
-                    computeConnectionStartBitrate(mediaDescs, trackBitrates)
+                    computeConnectionStartBitrate(mediaDescs, trackBitrates, connectionSetupTime)
                 } else {
                     null
                 }
+                if (connectionStartBitrate != null) {
+                    LKLog.i {
+                        "Applying x-google-start-bitrate=$connectionStartBitrate kbps " +
+                            "(connection setup ${connectionSetupTime?.inWholeMilliseconds} ms" +
+                            "${if (setupCompleted) "" else ", still connecting"})"
+                    }
+                }
                 for (mediaDesc in mediaDescs) {
                     if (mediaDesc.media.mediaType == "audio") {
                         // TODO
@@ -355,6 +376,35 @@ constructor(
         trackBitrates[TrackBitrateInfoKey.Transceiver(transceiver)] = trackBitrateInfo
     }
 
+    /**
+     * Records when the connection attempt that created this transport began, as
+     * [android.os.SystemClock.elapsedRealtime] milliseconds. A video offer created before the
+     * setup completes uses the time elapsed since then as the setup time.
+     */
+    fun setConnectStartedAt(elapsedRealtimeMs: Long) {
+        connectStartedAtMs = elapsedRealtimeMs
+    }
+
+    /**
+     * Records how long this peer connection took to set up. Called once, after the initial
+     * connect succeeds; resumes and ICE restarts keep the estimator and never call this.
+     */
+    fun setConnectionSetupTime(setupTime: Duration) {
+        connectionSetupTime = setupTime
+    }
+
+    /**
+     * The setup time the start bitrate hint is derived from: the completed setup time once the
+     * primary transport has connected, or, for a video offer created before then (an app that
+     * publishes as soon as the join completes), the time the connection has been setting up so
+     * far. That is a lower bound on the eventual setup time, so it can only err toward the
+     * 1 Mbps ceiling.
+     */
+    private fun connectionSetupTimeForOffer(): Duration? {
+        return connectionSetupTime
+            ?: connectStartedAtMs?.let { startedAtMs -> (SystemClock.elapsedRealtime() - startedAtMs).milliseconds }
+    }
+
     suspend fun isConnected(): Boolean {
         return launchRTCIfNotClosed {
             peerConnection.isConnected()
@@ -476,9 +526,24 @@ private const val startBitrateMultiplier = 0.9
 /** Maximum x-google-start-bitrate in kbps. 1 Mbps prevents BWE from starting too aggressively. */
 private const val maxStartBitrateKbps = 1000L
 
-/** Minimum target bitrate in kbps to apply start bitrate hint. Below this, the hint hurts more than it helps. */
+/**
+ * Minimum x-google-start-bitrate in kbps: libwebrtc's own default starting estimate. A target
+ * below this gets no hint, since seeding above the real capacity costs more than the ramp it
+ * saves, and a slow connection is seeded no higher than this.
+ */
 private const val minTargetBitrateKbps = 300L
 
+/**
+ * Connection setup times bounding the ramp in [computeTrackStartBitrate]: at or below the first
+ * the hint is capped at [maxStartBitrateKbps], at or above the second at [minTargetBitrateKbps].
+ * Measured on shaped links (CLT-3380): healthy links set up in under 860 ms, a 1 Mbps link with
+ * 150 ms RTT took 1.1–1.7 s, a 500 kbps link ~2.3 s at the median, and a 300 kbps link never
+ * under 3 s. The first sits above the 1 Mbps link, which a lower seed only slows down, and the
+ * second above the 500 kbps link's median, so only links that cannot carry more land at the floor.
+ */
+private val setupTimeForMaxBitrate = 1500.milliseconds
+private val setupTimeForMinBitrate = 3500.milliseconds
+
 @VisibleForTesting
 internal fun ensureCodecBitrates(
     media: MediaDescription,
@@ -549,23 +614,29 @@ internal fun ensureCodecBitrates(
 private fun computeConnectionStartBitrate(
     mediaDescriptions: Collection<MediaDescription>,
     trackBitrates: Map<TrackBitrateInfoKey, TrackBitrateInfo>,
+    connectionSetupTime: Duration?,
 ): Long? {
     // Use only video m-sections in the current SDP. trackBitrates can contain
     // stale entries after unpublish, and those must not affect the connection hint.
     return mediaDescriptions
         .asSequence()
         .filter { media -> media.media.mediaType == "video" }
         .mapNotNull { media -> findTrackCodecBitrateInfo(media, trackBitrates)?.trackBitrateInfo }
-        .mapNotNull(::computeTrackStartBitrate)
+        .mapNotNull { trackBitrateInfo -> computeTrackStartBitrate(trackBitrateInfo, connectionSetupTime) }
         .maxOrNull()
 }
 
 /**
  * @suppress
  */
 @VisibleForTesting
-internal fun computeConnectionStartBitrate(trackBitrates: Collection<TrackBitrateInfo>): Long? {
-    return trackBitrates.mapNotNull(::computeTrackStartBitrate).maxOrNull()
+internal fun computeConnectionStartBitrate(
+    trackBitrates: Collection<TrackBitrateInfo>,
+    connectionSetupTime: Duration? = null,
+): Long? {
+    return trackBitrates
+        .mapNotNull { trackBitrateInfo -> computeTrackStartBitrate(trackBitrateInfo, connectionSetupTime) }
+        .maxOrNull()
 }
 
 private data class TrackCodecBitrateInfo(
@@ -596,18 +667,31 @@ private fun findTrackCodecBitrateInfo(
     return null
 }
 
-private fun computeTrackStartBitrate(trackBr: TrackBitrateInfo): Long? {
+private fun computeTrackStartBitrate(trackBr: TrackBitrateInfo, connectionSetupTime: Duration?): Long? {
     if (trackBr.targetBitrateKbps < minTargetBitrateKbps) {
         return null
     }
 
-    // TODO: dynamically adjust start bitrate based on network conditions, such as
-    // using the previous BWE estimate.
+    // Connection setup time (signaling join plus ICE/DTLS) is the only network signal there is
+    // before the first video offer, since libwebrtc cannot probe the path until a video sender
+    // exists. It grows with round-trip time and loss, which also mark the links where a 1 Mbps
+    // seed overshoots, so the cap ramps linearly from maxStartBitrateKbps at
+    // setupTimeForMaxBitrate down to minTargetBitrateKbps at setupTimeForMinBitrate. Without a
+    // setup time (an offer before the connect completed) the cap stays at the 1 Mbps ceiling.
+    val capKbps = connectionSetupTime?.let { setupTime ->
+        val fast = setupTimeForMaxBitrate.inWholeMilliseconds.toDouble()
+        val slow = setupTimeForMinBitrate.inWholeMilliseconds.toDouble()
+        val ramp = ((setupTime.inWholeMilliseconds - fast) / (slow - fast)).coerceIn(0.0, 1.0)
+        (maxStartBitrateKbps - ramp * (maxStartBitrateKbps - minTargetBitrateKbps)).roundToLong()
+    } ?: maxStartBitrateKbps
+
     val calculatedStartBitrate = (trackBr.targetBitrateKbps * startBitrateMultiplier).roundToLong()
-    return if (trackBr.isScreenShare) {
+    // Screen share is exempt from the 1 Mbps ceiling, but once the cap is below it, a connection
+    // that slow cannot carry an uncapped screen-share seed either.
+    return if (trackBr.isScreenShare && capKbps >= maxStartBitrateKbps) {
         calculatedStartBitrate
     } else {
-        minOf(calculatedStartBitrate, maxStartBitrateKbps)
+        minOf(calculatedStartBitrate, capKbps)
     }
 }
 

livekit-android-sdk/src/main/java/io/livekit/android/room/RTCEngine.kt

@@ -128,6 +128,12 @@ internal constructor(
 ) : SignalClient.Listener {
     internal var listener: Listener? = null
 
+    /**
+     * When the current connection attempt began, taken at the top of [joinImpl]. Cleared once the
+     * primary transport connects, so the attempt is timed exactly once.
+     */
+    private var connectStartedAtMs: Long? = null
+
     /**
      * Reflects the combined connection state of SignalClient and primary PeerConnection.
      */
@@ -140,6 +146,9 @@ internal constructor(
         when (newVal) {
             ConnectionState.CONNECTED -> {
                 signalSessionState = SignalSessionState(ended = false)
+                if (oldVal != ConnectionState.RESUMING) {
+                    recordConnectionSetupTime()
+                }
                 if (oldVal == ConnectionState.DISCONNECTED || oldVal == ConnectionState.CONNECTING) {
                     LKLog.d { "primary ICE connected" }
                     listener?.onEngineConnected()
@@ -266,12 +275,30 @@ internal constructor(
         return joinImpl(url, token, options, roomOptions)
     }
 
+    /**
+     * Hands the time from the start of [joinImpl] to the primary transport connecting to the
+     * publisher, which lowers the start bitrate hint for a slow connection. Runs for the initial
+     * join and for a full reconnect, which both go through [joinImpl] and build a new publisher;
+     * a resume keeps its peer connections and their estimator, so it never records one. The
+     * publisher also knows when the attempt began, so a video offer created before this fires
+     * (an app publishing as soon as the join completes) uses the time elapsed so far.
+     */
+    private fun recordConnectionSetupTime() {
+        val startedAtMs = connectStartedAtMs ?: return
+        connectStartedAtMs = null
+        val setupTime = (SystemClock.elapsedRealtime() - startedAtMs).milliseconds
+        LKLog.i { "connection setup took ${setupTime.inWholeMilliseconds} ms" }
+        publisher?.setConnectionSetupTime(setupTime)
+    }
+
     suspend fun joinImpl(
         url: String,
         token: String,
         options: ConnectOptions,
         roomOptions: RoomOptions,
     ): JoinResponse = coroutineScope {
+        val startedAtMs = SystemClock.elapsedRealtime()
+        connectStartedAtMs = startedAtMs
         if (connectionState == ConnectionState.DISCONNECTED) {
             connectionState = ConnectionState.CONNECTING
         }
@@ -298,6 +325,9 @@ internal constructor(
         isSubscriberPrimary = joinResponse.subscriberPrimary
 
         configure(joinResponse, options)
+        // The publisher created above needs the attempt's start time before its first offer, in
+        // case video is published before the primary transport connects.
+        publisher?.setConnectStartedAt(startedAtMs)
 
         // Subscriber-primary defers the publisher PC until something is published. After a full
         // reconnect `hasPublished` is still set, so re-negotiate here — otherwise the ICE wait

livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt

@@ -25,6 +25,7 @@ import org.junit.Assert.assertEquals
 import org.junit.Assert.assertNotNull
 import org.junit.Assert.assertNull
 import org.junit.Test
+import kotlin.time.Duration.Companion.milliseconds
 
 class SdpMungingTest {
 
@@ -143,6 +144,56 @@ class SdpMungingTest {
         assertEquals(4500L, startBitrate)
     }
 
+    @Test
+    fun startBitrateRampsDownWithConnectionSetupTimeTest() {
+        // A 3 Mbps camera target, so only the cap moves.
+        val camera = listOf(TrackBitrateInfo(codec = "VP8", targetBitrateKbps = 3000L))
+        fun startBitrateAt(setupMs: Long) = computeConnectionStartBitrate(camera, setupMs.milliseconds)
+
+        assertEquals("instant setup keeps the ceiling", 1000L, startBitrateAt(0))
+        assertEquals("unshaped baseline keeps the ceiling", 1000L, startBitrateAt(471))
+        assertEquals("1 Mbps link median keeps the ceiling", 1000L, startBitrateAt(1273))
+        assertEquals("the fast anchor keeps the ceiling", 1000L, startBitrateAt(1500))
+        assertEquals("1 Mbps link slow attempt barely moves", 922L, startBitrateAt(1724))
+        assertEquals("500 kbps link median lands mid-ramp", 708L, startBitrateAt(2334))
+        assertEquals("midpoint of the ramp", 650L, startBitrateAt(2500))
+        assertEquals("300 kbps link fastest attempt", 442L, startBitrateAt(3093))
+        assertEquals("the slow anchor reaches the floor", 300L, startBitrateAt(3500))
+        assertEquals("anything slower stays at the floor", 300L, startBitrateAt(18_131))
+
+        assertEquals(
+            "90% of the target still wins when it is lower than the cap",
+            450L,
+            computeConnectionStartBitrate(
+                listOf(TrackBitrateInfo(codec = "VP8", targetBitrateKbps = 500L)),
+                2500.milliseconds,
+            ),
+        )
+        assertEquals(
+            "the hint is still written at the floor rather than skipped",
+            270L,
+            computeConnectionStartBitrate(
+                listOf(TrackBitrateInfo(codec = "VP8", targetBitrateKbps = 300L)),
+                3500.milliseconds,
+            ),
+        )
+        assertEquals(
+            "no setup time keeps today's cap",
+            1000L,
+            computeConnectionStartBitrate(camera, connectionSetupTime = null),
+        )
+    }
+
+    @Test
+    fun slowConnectionSetupCapsScreenShareTooTest() {
+        val screenShare = listOf(TrackBitrateInfo(codec = "VP8", targetBitrateKbps = 3000L, isScreenShare = true))
+        fun startBitrateAt(setupMs: Long) = computeConnectionStartBitrate(screenShare, setupMs.milliseconds)
+
+        assertEquals("a fast setup leaves screen share uncapped", 2700L, startBitrateAt(1273))
+        assertEquals("below the ceiling the cap applies to it too", 650L, startBitrateAt(2500))
+        assertEquals("the slowest setups seed it at the floor", 300L, startBitrateAt(4061))
+    }
+
     companion object {
         const val NO_DD_DESCRIPTION = "v=0\n" +
             "o=- 3682890773448528616 3 IN IP4 127.0.0.1\n" +