diff --git a/.changeset/scale-video-start-bitrate-by-connection-setup-time.md b/.changeset/scale-video-start-bitrate-by-connection-setup-time.md new file mode 100644 index 00000000..22b5d33e --- /dev/null +++ b/.changeset/scale-video-start-bitrate-by-connection-setup-time.md @@ -0,0 +1,5 @@ +--- +"client-sdk-android": patch +--- + +Scale the `x-google-start-bitrate` hint by connection setup time: the 1 Mbps camera cap now applies to connections that set up within 1.5 s and ramps linearly down to 300 kbps at 3.5 s or slower, with screen share capped the same way once the cap is below 1 Mbps. diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt index c4a3b004..6778e7ff 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt @@ -18,6 +18,7 @@ package io.livekit.android.room import android.javax.sdp.MediaDescription import android.javax.sdp.SdpFactory +import android.os.SystemClock import androidx.annotation.VisibleForTesting import dagger.assisted.Assisted import dagger.assisted.AssistedFactory @@ -63,6 +64,8 @@ import kotlin.contracts.ExperimentalContracts import kotlin.contracts.InvocationKind import kotlin.contracts.contract import kotlin.math.roundToLong +import kotlin.time.Duration +import kotlin.time.Duration.Companion.milliseconds /** * @suppress @@ -99,6 +102,15 @@ constructor( // available through data-channel/audio-only offers and consume it only after a // local video m-section successfully gets the hint. private var hasAppliedVideoStartBitrate = false + + // When the connection attempt that created this transport began, and how long it took to + // set up once the primary transport connected. computeTrackStartBitrate lowers the hint for + // slow connections; until the setup completes, the time elapsed so far stands in for it. + @Volatile + private var connectStartedAtMs: Long? = null + + @Volatile + private var connectionSetupTime: Duration? = null private var isClosed = AtomicBoolean(false) private val latestOfferId = AtomicInteger(0) @@ -217,11 +229,20 @@ constructor( // consume the video start hint. When the first video offer is created, use // one connection-level value across all video m-sections so libwebrtc's // last-writer-wins handling cannot depend on SDP m-section order. + val setupCompleted = connectionSetupTime != null + val connectionSetupTime = connectionSetupTimeForOffer() val connectionStartBitrate = if (!hasAppliedVideoStartBitrate) { - computeConnectionStartBitrate(mediaDescs, trackBitrates) + computeConnectionStartBitrate(mediaDescs, trackBitrates, connectionSetupTime) } else { null } + if (connectionStartBitrate != null) { + LKLog.i { + "Applying x-google-start-bitrate=$connectionStartBitrate kbps " + + "(connection setup ${connectionSetupTime?.inWholeMilliseconds} ms" + + "${if (setupCompleted) "" else ", still connecting"})" + } + } for (mediaDesc in mediaDescs) { if (mediaDesc.media.mediaType == "audio") { // TODO @@ -355,6 +376,35 @@ constructor( trackBitrates[TrackBitrateInfoKey.Transceiver(transceiver)] = trackBitrateInfo } + /** + * Records when the connection attempt that created this transport began, as + * [android.os.SystemClock.elapsedRealtime] milliseconds. A video offer created before the + * setup completes uses the time elapsed since then as the setup time. + */ + fun setConnectStartedAt(elapsedRealtimeMs: Long) { + connectStartedAtMs = elapsedRealtimeMs + } + + /** + * Records how long this peer connection took to set up. Called once, after the initial + * connect succeeds; resumes and ICE restarts keep the estimator and never call this. + */ + fun setConnectionSetupTime(setupTime: Duration) { + connectionSetupTime = setupTime + } + + /** + * The setup time the start bitrate hint is derived from: the completed setup time once the + * primary transport has connected, or, for a video offer created before then (an app that + * publishes as soon as the join completes), the time the connection has been setting up so + * far. That is a lower bound on the eventual setup time, so it can only err toward the + * 1 Mbps ceiling. + */ + private fun connectionSetupTimeForOffer(): Duration? { + return connectionSetupTime + ?: connectStartedAtMs?.let { startedAtMs -> (SystemClock.elapsedRealtime() - startedAtMs).milliseconds } + } + suspend fun isConnected(): Boolean { return launchRTCIfNotClosed { peerConnection.isConnected() @@ -476,9 +526,24 @@ private const val startBitrateMultiplier = 0.9 /** Maximum x-google-start-bitrate in kbps. 1 Mbps prevents BWE from starting too aggressively. */ private const val maxStartBitrateKbps = 1000L -/** Minimum target bitrate in kbps to apply start bitrate hint. Below this, the hint hurts more than it helps. */ +/** + * Minimum x-google-start-bitrate in kbps: libwebrtc's own default starting estimate. A target + * below this gets no hint, since seeding above the real capacity costs more than the ramp it + * saves, and a slow connection is seeded no higher than this. + */ private const val minTargetBitrateKbps = 300L +/** + * Connection setup times bounding the ramp in [computeTrackStartBitrate]: at or below the first + * the hint is capped at [maxStartBitrateKbps], at or above the second at [minTargetBitrateKbps]. + * Measured on shaped links (CLT-3380): healthy links set up in under 860 ms, a 1 Mbps link with + * 150 ms RTT took 1.1–1.7 s, a 500 kbps link ~2.3 s at the median, and a 300 kbps link never + * under 3 s. The first sits above the 1 Mbps link, which a lower seed only slows down, and the + * second above the 500 kbps link's median, so only links that cannot carry more land at the floor. + */ +private val setupTimeForMaxBitrate = 1500.milliseconds +private val setupTimeForMinBitrate = 3500.milliseconds + @VisibleForTesting internal fun ensureCodecBitrates( media: MediaDescription, @@ -549,6 +614,7 @@ internal fun ensureCodecBitrates( private fun computeConnectionStartBitrate( mediaDescriptions: Collection, trackBitrates: Map, + connectionSetupTime: Duration?, ): Long? { // Use only video m-sections in the current SDP. trackBitrates can contain // stale entries after unpublish, and those must not affect the connection hint. @@ -556,7 +622,7 @@ private fun computeConnectionStartBitrate( .asSequence() .filter { media -> media.media.mediaType == "video" } .mapNotNull { media -> findTrackCodecBitrateInfo(media, trackBitrates)?.trackBitrateInfo } - .mapNotNull(::computeTrackStartBitrate) + .mapNotNull { trackBitrateInfo -> computeTrackStartBitrate(trackBitrateInfo, connectionSetupTime) } .maxOrNull() } @@ -564,8 +630,13 @@ private fun computeConnectionStartBitrate( * @suppress */ @VisibleForTesting -internal fun computeConnectionStartBitrate(trackBitrates: Collection): Long? { - return trackBitrates.mapNotNull(::computeTrackStartBitrate).maxOrNull() +internal fun computeConnectionStartBitrate( + trackBitrates: Collection, + connectionSetupTime: Duration? = null, +): Long? { + return trackBitrates + .mapNotNull { trackBitrateInfo -> computeTrackStartBitrate(trackBitrateInfo, connectionSetupTime) } + .maxOrNull() } private data class TrackCodecBitrateInfo( @@ -596,18 +667,31 @@ private fun findTrackCodecBitrateInfo( return null } -private fun computeTrackStartBitrate(trackBr: TrackBitrateInfo): Long? { +private fun computeTrackStartBitrate(trackBr: TrackBitrateInfo, connectionSetupTime: Duration?): Long? { if (trackBr.targetBitrateKbps < minTargetBitrateKbps) { return null } - // TODO: dynamically adjust start bitrate based on network conditions, such as - // using the previous BWE estimate. + // Connection setup time (signaling join plus ICE/DTLS) is the only network signal there is + // before the first video offer, since libwebrtc cannot probe the path until a video sender + // exists. It grows with round-trip time and loss, which also mark the links where a 1 Mbps + // seed overshoots, so the cap ramps linearly from maxStartBitrateKbps at + // setupTimeForMaxBitrate down to minTargetBitrateKbps at setupTimeForMinBitrate. Without a + // setup time (an offer before the connect completed) the cap stays at the 1 Mbps ceiling. + val capKbps = connectionSetupTime?.let { setupTime -> + val fast = setupTimeForMaxBitrate.inWholeMilliseconds.toDouble() + val slow = setupTimeForMinBitrate.inWholeMilliseconds.toDouble() + val ramp = ((setupTime.inWholeMilliseconds - fast) / (slow - fast)).coerceIn(0.0, 1.0) + (maxStartBitrateKbps - ramp * (maxStartBitrateKbps - minTargetBitrateKbps)).roundToLong() + } ?: maxStartBitrateKbps + val calculatedStartBitrate = (trackBr.targetBitrateKbps * startBitrateMultiplier).roundToLong() - return if (trackBr.isScreenShare) { + // Screen share is exempt from the 1 Mbps ceiling, but once the cap is below it, a connection + // that slow cannot carry an uncapped screen-share seed either. + return if (trackBr.isScreenShare && capKbps >= maxStartBitrateKbps) { calculatedStartBitrate } else { - minOf(calculatedStartBitrate, maxStartBitrateKbps) + minOf(calculatedStartBitrate, capKbps) } } diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/RTCEngine.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/RTCEngine.kt index bc4c8235..c041ac66 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/RTCEngine.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/RTCEngine.kt @@ -128,6 +128,12 @@ internal constructor( ) : SignalClient.Listener { internal var listener: Listener? = null + /** + * When the current connection attempt began, taken at the top of [joinImpl]. Cleared once the + * primary transport connects, so the attempt is timed exactly once. + */ + private var connectStartedAtMs: Long? = null + /** * Reflects the combined connection state of SignalClient and primary PeerConnection. */ @@ -140,6 +146,9 @@ internal constructor( when (newVal) { ConnectionState.CONNECTED -> { signalSessionState = SignalSessionState(ended = false) + if (oldVal != ConnectionState.RESUMING) { + recordConnectionSetupTime() + } if (oldVal == ConnectionState.DISCONNECTED || oldVal == ConnectionState.CONNECTING) { LKLog.d { "primary ICE connected" } listener?.onEngineConnected() @@ -266,12 +275,30 @@ internal constructor( return joinImpl(url, token, options, roomOptions) } + /** + * Hands the time from the start of [joinImpl] to the primary transport connecting to the + * publisher, which lowers the start bitrate hint for a slow connection. Runs for the initial + * join and for a full reconnect, which both go through [joinImpl] and build a new publisher; + * a resume keeps its peer connections and their estimator, so it never records one. The + * publisher also knows when the attempt began, so a video offer created before this fires + * (an app publishing as soon as the join completes) uses the time elapsed so far. + */ + private fun recordConnectionSetupTime() { + val startedAtMs = connectStartedAtMs ?: return + connectStartedAtMs = null + val setupTime = (SystemClock.elapsedRealtime() - startedAtMs).milliseconds + LKLog.i { "connection setup took ${setupTime.inWholeMilliseconds} ms" } + publisher?.setConnectionSetupTime(setupTime) + } + suspend fun joinImpl( url: String, token: String, options: ConnectOptions, roomOptions: RoomOptions, ): JoinResponse = coroutineScope { + val startedAtMs = SystemClock.elapsedRealtime() + connectStartedAtMs = startedAtMs if (connectionState == ConnectionState.DISCONNECTED) { connectionState = ConnectionState.CONNECTING } @@ -298,6 +325,9 @@ internal constructor( isSubscriberPrimary = joinResponse.subscriberPrimary configure(joinResponse, options) + // The publisher created above needs the attempt's start time before its first offer, in + // case video is published before the primary transport connects. + publisher?.setConnectStartedAt(startedAtMs) // Subscriber-primary defers the publisher PC until something is published. After a full // reconnect `hasPublished` is still set, so re-negotiate here — otherwise the ICE wait diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt index 2c8cb83e..18618d7c 100644 --- a/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt +++ b/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt @@ -25,6 +25,7 @@ import org.junit.Assert.assertEquals import org.junit.Assert.assertNotNull import org.junit.Assert.assertNull import org.junit.Test +import kotlin.time.Duration.Companion.milliseconds class SdpMungingTest { @@ -143,6 +144,56 @@ class SdpMungingTest { assertEquals(4500L, startBitrate) } + @Test + fun startBitrateRampsDownWithConnectionSetupTimeTest() { + // A 3 Mbps camera target, so only the cap moves. + val camera = listOf(TrackBitrateInfo(codec = "VP8", targetBitrateKbps = 3000L)) + fun startBitrateAt(setupMs: Long) = computeConnectionStartBitrate(camera, setupMs.milliseconds) + + assertEquals("instant setup keeps the ceiling", 1000L, startBitrateAt(0)) + assertEquals("unshaped baseline keeps the ceiling", 1000L, startBitrateAt(471)) + assertEquals("1 Mbps link median keeps the ceiling", 1000L, startBitrateAt(1273)) + assertEquals("the fast anchor keeps the ceiling", 1000L, startBitrateAt(1500)) + assertEquals("1 Mbps link slow attempt barely moves", 922L, startBitrateAt(1724)) + assertEquals("500 kbps link median lands mid-ramp", 708L, startBitrateAt(2334)) + assertEquals("midpoint of the ramp", 650L, startBitrateAt(2500)) + assertEquals("300 kbps link fastest attempt", 442L, startBitrateAt(3093)) + assertEquals("the slow anchor reaches the floor", 300L, startBitrateAt(3500)) + assertEquals("anything slower stays at the floor", 300L, startBitrateAt(18_131)) + + assertEquals( + "90% of the target still wins when it is lower than the cap", + 450L, + computeConnectionStartBitrate( + listOf(TrackBitrateInfo(codec = "VP8", targetBitrateKbps = 500L)), + 2500.milliseconds, + ), + ) + assertEquals( + "the hint is still written at the floor rather than skipped", + 270L, + computeConnectionStartBitrate( + listOf(TrackBitrateInfo(codec = "VP8", targetBitrateKbps = 300L)), + 3500.milliseconds, + ), + ) + assertEquals( + "no setup time keeps today's cap", + 1000L, + computeConnectionStartBitrate(camera, connectionSetupTime = null), + ) + } + + @Test + fun slowConnectionSetupCapsScreenShareTooTest() { + val screenShare = listOf(TrackBitrateInfo(codec = "VP8", targetBitrateKbps = 3000L, isScreenShare = true)) + fun startBitrateAt(setupMs: Long) = computeConnectionStartBitrate(screenShare, setupMs.milliseconds) + + assertEquals("a fast setup leaves screen share uncapped", 2700L, startBitrateAt(1273)) + assertEquals("below the ceiling the cap applies to it too", 650L, startBitrateAt(2500)) + assertEquals("the slowest setups seed it at the floor", 300L, startBitrateAt(4061)) + } + companion object { const val NO_DD_DESCRIPTION = "v=0\n" + "o=- 3682890773448528616 3 IN IP4 127.0.0.1\n" +