test(moq): prove bounded slow-consumer recovery
- commit
- 17980ad67610543ec48227ad0528c899083ab783
- parent
- 8010a8e144ea59a036da2e4a0aac11b508bf5e4a
- parent
- 11c1a99510b1ab847dc6d273fece90f50d6463d9
- author
- FelineStateMachine <dami.etoile@gmail.com>
| M | crates/ | 219 | +++++++++++++++++++++++++++++- |
| M | scripts/ | 39 | ++++- |
| M | scripts/ | 26 | +++- |
@@ -78,6 +78,15 @@ struct Args {
7878 /// Correlation identifier for `--keyframe-smoke` host evidence.7979 8080 keyframe_request_id: u64,81+ /// Stop reading upstream MoQ after the first configured IDR, then require82+ /// the transport to cancel that stale GOP and resume at a newer configured83+ /// IDR within a bounded interval. Intended for latency-growth proofs.84+ 85+86+87+88+ )]89+ slow_consumer_ms: Option<u64>,8190 /// Require gamepad negotiation and emit one bounded non-neutral snapshot8291 /// followed by neutral. Intended for evtest-backed uinput proof.8392 @@ -128,6 +137,8 @@ impl MediaTransport {
128137 129138 const KEYFRAME_SMOKE_REQUEST_AFTER_FRAMES: u32 = 3;130139 const KEYFRAME_SMOKE_MINIMUM_FRAMES: u32 = KEYFRAME_SMOKE_REQUEST_AFTER_FRAMES + 1;140+const SLOW_CONSUMER_MINIMUM_FRAMES: u32 = 2;141+const SLOW_CONSUMER_RECOVERY_TIMEOUT: Duration = Duration::from_secs(2);131142 const MAX_INVITATION_FILE_BYTES: u64 = (MAX_INVITATION_TOKEN_LEN + 2) as u64;132143 133144 @@ -928,6 +939,7 @@ struct AcceptedMedia {
928939 width: u16,929940 height: u16,930941 sequence: u64,942+ capture_timestamp_us: u64,931943 payload_len: usize,932944 v3_group_id: Option<u64>,933945 }@@ -939,6 +951,7 @@ impl From<MediaFrame> for AcceptedMedia {
939951 width: frame.header.width,940952 height: frame.header.height,941953 sequence: frame.header.sequence,954+ capture_timestamp_us: frame.header.capture_timestamp_us,942955 payload_len: frame.payload.len(),943956 v3_group_id: None,944957 }@@ -952,6 +965,7 @@ impl From<MediaObjectV3> for AcceptedMedia {
952965 width: object.header.width,953966 height: object.header.height,954967 sequence: object.header.sequence,968+ capture_timestamp_us: object.header.capture_timestamp_us,955969 payload_len: object.payload.len(),956970 v3_group_id: Some(object.header.group_id),957971 }@@ -973,6 +987,10 @@ async fn main() -> Result<()> {
973987 !args.keyframe_smoke || args.frames >= KEYFRAME_SMOKE_MINIMUM_FRAMES,974988 "--keyframe-smoke requires at least {KEYFRAME_SMOKE_MINIMUM_FRAMES} accepted frames"975989 );990+ ensure!(991+ args.slow_consumer_ms.is_none() || args.frames >= SLOW_CONSUMER_MINIMUM_FRAMES,992+ "--slow-consumer-ms requires at least {SLOW_CONSUMER_MINIMUM_FRAMES} accepted frames"993+ );976994 977995 let secret = match args.identity.as_deref() {978996 Some(path) => identity::load(path).context("loading persistent probe identity")?,@@ -1199,13 +1217,31 @@ async fn main() -> Result<()> {
11991217 let mut keyframe_recovery_micros = None;12001218 let mut keyframe_request_group_id = None;12011219 let mut keyframe_request_last_sequence = None;1220+ let mut slow_consumer_stalled = false;1221+ let mut slow_consumer_pending_recovery = false;1222+ let mut slow_consumer_verified = false;1223+ let mut slow_consumer_group_id = None;1224+ let mut slow_consumer_last_sequence = None;1225+ let mut slow_consumer_capture_timestamp_us = None;1226+ let mut slow_consumer_cancelled_groups_before = None;1227+ let mut slow_consumer_resumed_at: Option<Instant> = None;1228+ let mut slow_consumer_recovery_micros = None;1229+ let mut slow_consumer_cancellation_delta = None;1230+ let mut slow_consumer_group_advance = None;1231+ let mut slow_consumer_sequence_advance = None;1232+ let mut slow_consumer_capture_advance_micros = None;1233+ let mut slow_consumer_input_ack_micros = None;12021234 12031235 while received < args.frames {12041236 let (frame, recovery_frame): (AcceptedMedia, bool) = match media_transport {12051237 MediaTransport::UpstreamMoq => loop {1238+ let media_wait = if slow_consumer_pending_recovery {1239+ SLOW_CONSUMER_RECOVERY_TIMEOUT1240+ } else {1241+ Duration::from_secs(args.timeout_seconds).saturating_add(Duration::from_secs(1))1242+ };12061243 let outcome = tokio::time::timeout(1207- Duration::from_secs(args.timeout_seconds)1208- .saturating_add(Duration::from_secs(1)),1244+ media_wait,12091245 moq_receiver12101246 .as_mut()12111247 .expect("upstream MoQ receiver is present")@@ -1223,9 +1259,9 @@ async fn main() -> Result<()> {
12231259 group_sequence,12241260 recovery,12251261 } => {1226- let requested_recovery =1262+ let requested_keyframe_recovery =12271263 keyframe_request_sent && !keyframe_recovery_verified;1228- if requested_recovery {1264+ if requested_keyframe_recovery {12291265 ensure!(12301266 recovery12311267 && frame.header.flags.contains(FrameFlags::KEYFRAME)@@ -1247,16 +1283,101 @@ async fn main() -> Result<()> {
12471283 });12481284 keyframe_recovery_verified = true;12491285 }1286+ let slow_consumer_recovery = slow_consumer_pending_recovery;1287+ if slow_consumer_recovery {1288+ ensure!(1289+ recovery1290+ && frame.header.flags.contains(FrameFlags::KEYFRAME)1291+ && frame.header.flags.contains(FrameFlags::CODEC_CONFIG),1292+ "MoQ slow-consumer recovery did not begin at a configured IDR barrier"1293+ );1294+ let prior_group = slow_consumer_group_id1295+ .context("slow-consumer group was not recorded")?;1296+ let group_advance = group_sequence.checked_sub(prior_group).context(1297+ "MoQ slow-consumer recovery did not advance the native group",1298+ )?;1299+ ensure!(1300+ group_advance > 0,1301+ "MoQ slow-consumer recovery did not advance the native group"1302+ );1303+ let prior_sequence = slow_consumer_last_sequence1304+ .context("slow-consumer sequence was not recorded")?;1305+ let sequence_advance =1306+ frame.header.sequence.checked_sub(prior_sequence).context(1307+ "MoQ slow-consumer recovery did not advance media sequence",1308+ )?;1309+ ensure!(1310+ sequence_advance > 0,1311+ "MoQ slow-consumer recovery did not advance media sequence"1312+ );1313+ let prior_capture_timestamp = slow_consumer_capture_timestamp_us1314+ .context("slow-consumer capture timestamp was not recorded")?;1315+ let capture_advance = frame1316+ .header1317+ .capture_timestamp_us1318+ .checked_sub(prior_capture_timestamp)1319+ .context(1320+ "MoQ slow-consumer recovery capture timestamp moved backward",1321+ )?;1322+ let minimum_capture_advance = args1323+ .slow_consumer_ms1324+ .expect("slow-consumer duration is configured")1325+ .saturating_mul(1_000)1326+ / 2;1327+ ensure!(1328+ capture_advance >= minimum_capture_advance,1329+ "MoQ slow-consumer recovery advanced capture time by only {capture_advance}us; required at least {minimum_capture_advance}us"1330+ );1331+ let cancelled_before = slow_consumer_cancelled_groups_before1332+ .context("slow-consumer cancellation count was not recorded")?;1333+ let cancelled_after = moq_receiver1334+ .as_ref()1335+ .expect("upstream MoQ receiver is present")1336+ .cancelled_groups;1337+ let cancellation_delta = cancelled_after1338+ .checked_sub(cancelled_before)1339+ .context("MoQ slow-consumer cancellation count moved backward")?;1340+ ensure!(1341+ cancellation_delta > 0,1342+ "MoQ slow-consumer recovery did not observe a cancelled stale group"1343+ );1344+ ensure!(1345+ moq_receiver1346+ .as_ref()1347+ .expect("upstream MoQ receiver is present")1348+ .historical_suffix_frames1349+ == 0,1350+ "MoQ slow-consumer recovery accepted historical delta suffix frames"1351+ );1352+ let recovery_micros = slow_consumer_resumed_at1353+ .context("slow-consumer resume instant was not recorded")?1354+ .elapsed()1355+ .as_micros();1356+ ensure!(1357+ recovery_micros <= SLOW_CONSUMER_RECOVERY_TIMEOUT.as_micros(),1358+ "MoQ slow-consumer recovery exceeded {}us",1359+ SLOW_CONSUMER_RECOVERY_TIMEOUT.as_micros()1360+ );1361+ slow_consumer_recovery_micros =1362+ Some(u64::try_from(recovery_micros).unwrap_or(u64::MAX));1363+ slow_consumer_cancellation_delta = Some(cancellation_delta);1364+ slow_consumer_group_advance = Some(group_advance);1365+ slow_consumer_sequence_advance = Some(sequence_advance);1366+ slow_consumer_capture_advance_micros = Some(capture_advance);1367+ slow_consumer_pending_recovery = false;1368+ slow_consumer_verified = true;1369+ }12501370 break (12511371 AcceptedMedia {12521372 flags: frame.header.flags,12531373 width: frame.header.width,12541374 height: frame.header.height,12551375 sequence: frame.header.sequence,1376+ capture_timestamp_us: frame.header.capture_timestamp_us,12561377 payload_len: frame.payload.len(),12571378 v3_group_id: Some(group_sequence),12581379 },1259- requested_recovery,1380+ requested_keyframe_recovery || slow_consumer_recovery,12601381 );12611382 }12621383 }@@ -1422,6 +1543,40 @@ async fn main() -> Result<()> {
14221543 bytes = bytes.saturating_add(frame.payload_len as u64);14231544 received += 1;14241545 1546+ if let Some(stall_ms) = args.slow_consumer_ms1547+ && !slow_consumer_stalled1548+ {1549+ slow_consumer_group_id = frame.v3_group_id;1550+ slow_consumer_last_sequence = Some(frame.sequence);1551+ slow_consumer_capture_timestamp_us = Some(frame.capture_timestamp_us);1552+ slow_consumer_cancelled_groups_before = Some(1553+ moq_receiver1554+ .as_ref()1555+ .expect("upstream MoQ receiver is present")1556+ .cancelled_groups,1557+ );1558+ let first_half = Duration::from_millis(stall_ms / 2);1559+ let second_half = Duration::from_millis(stall_ms).saturating_sub(first_half);1560+ tokio::time::sleep(first_half).await;1561+ let input_probe_started = Instant::now();1562+ write_input_event(&mut input_send, &InputEvent::Probe)1563+ .await1564+ .context("writing input probe during slow-consumer media stall")?;1565+ expected_ack = expected_ack.saturating_add(1);1566+ tokio::time::timeout(1567+ SLOW_CONSUMER_RECOVERY_TIMEOUT,1568+ read_expected_input_ack(&mut input_recv, args.timeout_seconds, expected_ack),1569+ )1570+ .await1571+ .context("input acknowledgement was blocked by the slow media consumer")??;1572+ slow_consumer_input_ack_micros =1573+ Some(u64::try_from(input_probe_started.elapsed().as_micros()).unwrap_or(u64::MAX));1574+ tokio::time::sleep(second_half).await;1575+ slow_consumer_resumed_at = Some(Instant::now());1576+ slow_consumer_pending_recovery = true;1577+ slow_consumer_stalled = true;1578+ }1579+14251580 if args.keyframe_smoke14261581 && !keyframe_request_sent14271582 && received == KEYFRAME_SMOKE_REQUEST_AFTER_FRAMES@@ -1480,6 +1635,10 @@ async fn main() -> Result<()> {
14801635 !args.keyframe_smoke || keyframe_recovery_verified,14811636 "probe did not verify requested keyframe recovery"14821637 );1638+ ensure!(1639+ args.slow_consumer_ms.is_none() || slow_consumer_verified,1640+ "probe did not verify bounded slow-consumer recovery"1641+ );14831642 let accepted_span = first_accepted_at14841643 .zip(last_accepted_at)14851644 .map_or(Duration::ZERO, |(first, last)| {@@ -1616,6 +1775,56 @@ async fn main() -> Result<()> {
16161775 println!("keyframe_request_id=not-requested");16171776 println!("keyframe_recovery_micros=not-requested");16181777 }1778+ if let Some(stall_ms) = args.slow_consumer_ms {1779+ println!("slow_consumer=ok");1780+ println!("slow_consumer_stall_ms={stall_ms}");1781+ println!("slow_consumer_first_post_stall=configured-idr");1782+ println!("slow_consumer_historical_suffix_frames=0");1783+ println!(1784+ "slow_consumer_recovery_micros={}",1785+ slow_consumer_recovery_micros1786+ .context("slow-consumer recovery latency was not measured")?1787+ );1788+ println!(1789+ "slow_consumer_cancellation_delta={}",1790+ slow_consumer_cancellation_delta1791+ .context("slow-consumer cancellation delta was not measured")?1792+ );1793+ println!(1794+ "slow_consumer_group_advance={}",1795+ slow_consumer_group_advance.context("slow-consumer group advance was not measured")?1796+ );1797+ println!(1798+ "slow_consumer_sequence_advance={}",1799+ slow_consumer_sequence_advance1800+ .context("slow-consumer sequence advance was not measured")?1801+ );1802+ println!(1803+ "slow_consumer_capture_advance_micros={}",1804+ slow_consumer_capture_advance_micros1805+ .context("slow-consumer capture advance was not measured")?1806+ );1807+ println!(1808+ "slow_consumer_input_ack_micros={}",1809+ slow_consumer_input_ack_micros1810+ .context("slow-consumer input acknowledgement was not measured")?1811+ );1812+ } else {1813+ for field in [1814+ "slow_consumer",1815+ "slow_consumer_stall_ms",1816+ "slow_consumer_first_post_stall",1817+ "slow_consumer_historical_suffix_frames",1818+ "slow_consumer_recovery_micros",1819+ "slow_consumer_cancellation_delta",1820+ "slow_consumer_group_advance",1821+ "slow_consumer_sequence_advance",1822+ "slow_consumer_capture_advance_micros",1823+ "slow_consumer_input_ack_micros",1824+ ] {1825+ println!("{field}=not-requested");1826+ }1827+ }16191828 match accepted_fps {16201829 Some(fps) => println!("accepted_fps={fps:.3}"),16211830 None => println!("accepted_fps=unknown"),@@ -448,6 +448,26 @@ validate_probe_log() {
448448 awk -F= '$1 == "moq_maximum_group_bytes" && $2 ~ /^[0-9]+$/ && $2 > 0 && $2 <= 33554432 { found=1 } END { exit !found }' \449449 "$log_path" || return 1450450 fi451+ if [[ "$media_transport" == "iroh-moq" && "$expected_request_id" == 1 ]]; then452+ grep -Fxq 'slow_consumer=ok' "$log_path" || return 1453+ grep -Fxq 'slow_consumer_stall_ms=1500' "$log_path" || return 1454+ grep -Fxq 'slow_consumer_first_post_stall=configured-idr' "$log_path" || return 1455+ grep -Fxq 'slow_consumer_historical_suffix_frames=0' "$log_path" || return 1456+ awk -F= '$1 == "slow_consumer_recovery_micros" && $2 ~ /^[0-9]+$/ && $2 <= 2000000 { found=1 } END { exit !found }' \457+ "$log_path" || return 1458+ awk -F= '$1 == "slow_consumer_cancellation_delta" && $2 ~ /^[0-9]+$/ && $2 > 0 { found=1 } END { exit !found }' \459+ "$log_path" || return 1460+ awk -F= '$1 == "slow_consumer_group_advance" && $2 ~ /^[0-9]+$/ && $2 > 0 { found=1 } END { exit !found }' \461+ "$log_path" || return 1462+ awk -F= '$1 == "slow_consumer_sequence_advance" && $2 ~ /^[0-9]+$/ && $2 > 0 { found=1 } END { exit !found }' \463+ "$log_path" || return 1464+ awk -F= '$1 == "slow_consumer_capture_advance_micros" && $2 ~ /^[0-9]+$/ && $2 >= 750000 { found=1 } END { exit !found }' \465+ "$log_path" || return 1466+ awk -F= '$1 == "slow_consumer_input_ack_micros" && $2 ~ /^[0-9]+$/ && $2 <= 2000000 { found=1 } END { exit !found }' \467+ "$log_path" || return 1468+ else469+ grep -Fxq 'slow_consumer=not-requested' "$log_path" || return 1470+ fi451471 }452472 453473 run_probe() {@@ -512,13 +532,16 @@ wait_for_log_line "$host_log" 'status=ready' "$host_pid" 'sigil' || die "host di
512532 host_node_id="$(sed -n 's/^node_id=//p' "$host_log" | tail -n 1)"513533 [[ "$host_node_id" == "$node_id" ]] || die "ready host node ID does not match its identity"514534 515-run_proof_probe \516- 1 \517- --node-id "$node_id" \518- --frames "$primary_frames" \519- --timeout-seconds "$probe_timeout_seconds" \520- --expect-size 1280x800 \521- >"$primary_log" 2>&1 &535+primary_probe_args=(536+ --node-id "$node_id"537+ --frames "$primary_frames"538+ --timeout-seconds "$probe_timeout_seconds"539+ --expect-size 1280x800540+)541+if [[ "$media_v1" == false && "$media_v2" == false && "$media_v3" == false ]]; then542+ primary_probe_args+=(--slow-consumer-ms 1500)543+fi544+run_proof_probe 1 "${primary_probe_args[@]}" >"$primary_log" 2>&1 &522545 primary_pid=$!523546 start_watchdog "$primary_pid" "$command_timeout_seconds" "$tmp_root/primary.timeout"524547 primary_watchdog_pid="$watchdog_pid"@@ -629,7 +652,7 @@ printf 'probe_binary=%s\n' "$probe_bin"
629652 printf 'probe_sha256=%s\n' "$(sha256_file "$probe_bin")"630653 printf 'node_id=%s\n' "$node_id"631654 printf 'primary_frames=%s\n' "$primary_frames"632-grep -E '^(transport|control_alpn|transport_alpn|first_configured_idr|frame_sequence|group_sequence|keyframe_recovery|keyframe_recovery_micros|keyframe_request_id|keyframes|sequence_gaps|media_objects_dropped|media_objects_late|moq_catalog|moq_group_capacity|moq_cancelled_groups|moq_group_gaps|moq_unrecovered_group_gaps|moq_maximum_group_objects|moq_maximum_group_bytes|moq_historical_suffix_frames|recovery_barrier|input_ack_micros|path_mode|path_rtt_ms)=' "$primary_log"655+grep -E '^(transport|control_alpn|transport_alpn|first_configured_idr|frame_sequence|group_sequence|keyframe_recovery|keyframe_recovery_micros|keyframe_request_id|keyframes|sequence_gaps|media_objects_dropped|media_objects_late|moq_catalog|moq_group_capacity|moq_cancelled_groups|moq_group_gaps|moq_unrecovered_group_gaps|moq_maximum_group_objects|moq_maximum_group_bytes|moq_historical_suffix_frames|recovery_barrier|slow_consumer|slow_consumer_stall_ms|slow_consumer_first_post_stall|slow_consumer_historical_suffix_frames|slow_consumer_recovery_micros|slow_consumer_cancellation_delta|slow_consumer_group_advance|slow_consumer_sequence_advance|slow_consumer_capture_advance_micros|slow_consumer_input_ack_micros|input_ack_micros|path_mode|path_rtt_ms)=' "$primary_log"633656 printf 'keyframe_request_correlation=%s\n' \634657 "$([[ "$media_v1" == false && "$media_v2" == false ]] && printf unique || printf not-requested)"635658 printf 'active_client_rejection=ok\n'@@ -404,6 +404,7 @@ validate_probe() {
404404 local transport="$2"405405 local size="$3"406406 local request_id="$4"407+ local slow_consumer_ms="${5:-}"407408 grep -Fxq probe=ok "$log"408409 grep -Fxq "dimensions=$size" "$log"409410 grep -Fxq "transport=$transport" "$log"@@ -422,6 +423,20 @@ validate_probe() {
422423 grep -Fxq moq_unrecovered_group_gaps=0 "$log"423424 grep -Fxq moq_historical_suffix_frames=0 "$log"424425 fi426+ if [[ -n "$slow_consumer_ms" ]]; then427+ grep -Fxq slow_consumer=ok "$log"428+ grep -Fxq "slow_consumer_stall_ms=$slow_consumer_ms" "$log"429+ grep -Fxq slow_consumer_first_post_stall=configured-idr "$log"430+ grep -Fxq slow_consumer_historical_suffix_frames=0 "$log"431+ awk -F= '$1 == "slow_consumer_recovery_micros" && $2 ~ /^[0-9]+$/ && $2 <= 2000000 { ok=1 } END { exit !ok }' "$log"432+ awk -F= '$1 == "slow_consumer_cancellation_delta" && $2 ~ /^[0-9]+$/ && $2 > 0 { ok=1 } END { exit !ok }' "$log"433+ awk -F= '$1 == "slow_consumer_group_advance" && $2 ~ /^[0-9]+$/ && $2 > 0 { ok=1 } END { exit !ok }' "$log"434+ awk -F= '$1 == "slow_consumer_sequence_advance" && $2 ~ /^[0-9]+$/ && $2 > 0 { ok=1 } END { exit !ok }' "$log"435+ awk -F= -v minimum="$((slow_consumer_ms * 500))" '$1 == "slow_consumer_capture_advance_micros" && $2 ~ /^[0-9]+$/ && $2 >= minimum { ok=1 } END { exit !ok }' "$log"436+ awk -F= '$1 == "slow_consumer_input_ack_micros" && $2 ~ /^[0-9]+$/ && $2 <= 2000000 { ok=1 } END { exit !ok }' "$log"437+ else438+ grep -Fxq slow_consumer=not-requested "$log"439+ fi425440 }426441 427442 wait_for_count() {@@ -447,6 +462,7 @@ run_probe_cycle() {
447462 local size="$3"448463 local request_id="$4"449464 local invitation_path="${5:-}"465+ local slow_consumer_ms="${6:-}"450466 local log="$raw/$mode-$transport-$request_id.log"451467 local command=(452468 "$probe"@@ -460,8 +476,9 @@ run_probe_cycle() {
460476 )461477 [[ "$transport" == grouped-v3 ]] && command+=(--media-v3)462478 [[ -n "$invitation_path" ]] && command+=(--invitation "$invitation_path")479+ [[ -n "$slow_consumer_ms" ]] && command+=(--slow-consumer-ms "$slow_consumer_ms")463480 timeout --signal=TERM --kill-after=5s 45s "${command[@]}" >"$log" 2>&1464- validate_probe "$log" "$transport" "$size" "$request_id"481+ validate_probe "$log" "$transport" "$size" "$request_id" "$slow_consumer_ms"465482 }466483 467484 validate_host_recovery() {@@ -499,7 +516,7 @@ grep -Fxq probe=ok "$evidence/fixed-capture.log"
499516 grep -Fxq dropped_after_encode_before_probe_consumer=0 "$evidence/fixed-capture.log"500517 501518 start_candidate fixed "$fixed_config"502-run_probe_cycle fixed iroh-moq 1280x800 1001 "$invitation"519+run_probe_cycle fixed iroh-moq 1280x800 1001 "$invitation" 1500503520 wait_for_count "$raw/fixed-host.log" 'MoQ control client released' 1504521 wait_for_count "$raw/fixed-host.log" 'input client released' 1505522 enrollment="$("$sigil" enrollment show --config "$fixed_config")"@@ -535,7 +552,8 @@ grep -Fxq probe=ok "$evidence/native-capture.log"
535552 grep -Fxq dropped_after_encode_before_probe_consumer=0 "$evidence/native-capture.log"536553 537554 start_candidate native "$native_config"538-for cycle in $(seq 1 10); do555+run_probe_cycle native iroh-moq "$native_size" 2001 "" 1500556+for cycle in $(seq 2 10); do539557 run_probe_cycle native iroh-moq "$native_size" "$((2000 + cycle))"540558 done541559 for cycle in $(seq 1 10); do@@ -624,6 +642,8 @@ fi
624642 echo "fixed_daemon_unique_sessions=$fixed_unique_sessions"625643 echo "native_daemon_unique_sessions=$native_unique_sessions"626644 echo "total_sessions=$total_sessions"645+ echo slow_media_consumer=pass646+ echo slow_media_consumer_resolutions=1280x800,"$native_size"627647 summarize_group fixed-iroh-moq628648 summarize_group fixed-grouped-v3629649 summarize_group native-iroh-moq