cat / goq
clone

turns a linux box into a private steam streaming appliance: plug in, tap your key, play from another computer

refactor: retire legacy v0/v1/v2 media tiers and the base64 JPEG fallback

Dami committed 5f090ad
Pre-alpha with no deployed peers, so the compatibility surface collapses
to the two paths the product actually uses: upstream MoQ (preferred) and
grouped-v3 (sole fallback tier).

Host: MEDIA_ALPN_V1/MEDIA_ALPN_V2 are no longer registered; serve_media
and serve_media_v2 (all of media_v2.rs) are deleted, with
media_frame_for_encoded moved next to its only consumer in moq.rs.

Client: the hardcoded use_v1 flag and the unreachable v0 leg (no hello,
single start byte) are gone, along with the v2/v1 fallback ladder, the
v1 object-reorder machinery, and the LegacyV0/ReliableStreamV1/
IndependentObjectsV2 transport variants. The negotiated control stream
is now threaded as a plain (non-Option) value.

Base64 fallback: per AGENTS.md 'Never reintroduce base64 media events;
use binary Tauri channels', the openh264-to-JPEG-to-base64 app.emit
path is deleted end to end - Rust decoder branch, FramePayload event,
--dev-force-jpeg flag, openh264/image/base64 dependencies, the portal
JPEG presenter, and the 'frame' event listener. Connecting now fails
fast with a clear error when the webview lacks WebCodecs H.264 support.

Probe: v1/v2 tiers, their CLI flags, and the duplicated v1 reorder
state machine are removed (-520 lines).

Protocol: MEDIA_ALPN_V1/V2, LEGACY_FRAME_ALPN_V0/LEGACY_INPUT_ALPN_V0,
and the v1/v2 stream IO (read/write_media_frame, read_media_object) are
deleted; MediaFrame stays as the MoQ object payload format.

The portal transport allowlist now mirrors the two remaining transports.
commit
5f090adffd773ab596726969758c545128078cf4
parent
d90cfb4b35cffc432ad92832997bf5c1b2c580d6
author
Dami <dami.etoile@gmail.com>
32 files changed, +304 -3474
MCargo.lock103----
Mcrates/sigil-host/src/bin/sigil-probe.rs540+-------------------
Mcrates/sigil-host/src/main.rs20+-
Mcrates/sigil-host/src/server/handlers.rs48--
Dcrates/sigil-host/src/server/media_v2.rs821------------------------------
Mcrates/sigil-host/src/server/mod.rs9+-
Mcrates/sigil-host/src/server/moq.rs27+
Mcrates/sigil-protocol/src/lib.rs19+-
Mcrates/sigil-protocol/src/media.rs144-----
Mportal/connection-state.mjs4+-
Mportal/connection-state.test.mjs3-
Mportal/enrollment-ui.test.mjs7-
Mportal/frame-session.mjs38+-
Mportal/frame-session.test.mjs27-
Mportal/index.html1-
Mportal/main.js21+-
Mportal/stream-runtime.mjs24-
Mportal/stream-runtime.test.mjs27+-
Mportal/video-capability.mjs5+-
Mportal/video-capability.test.mjs11+-
Mportal/video-pipeline.mjs297+++--------
Mportal/video-pipeline.test.mjs230+--------
Msrc-tauri/Cargo.toml3-
Msrc-tauri/src/commands/network.rs5+-
Msrc-tauri/src/commands/state.rs126+----
Msrc-tauri/src/media/audio_delivery.rs6+-
Msrc-tauri/src/media/connect.rs149++----
Msrc-tauri/src/media/frame_channel.rs44--
Msrc-tauri/src/media/input_delivery.rs106+---
Msrc-tauri/src/media/object_receiver.rs410+---------------
Msrc-tauri/src/media/transport.rs171+-----
Msrc-tauri/src/media/video_delivery.rs332+-----------
M Cargo.lock +0 -103
@@ -620,12 +620,6 @@ version = "1.5.0"
620620 source = "registry+https://github.com/rust-lang/crates.io-index"621621 checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"622622 623-[[package]]624-name = "byteorder-lite"625-version = "0.1.0"626-source = "registry+https://github.com/rust-lang/crates.io-index"627-checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"628-629623 [[package]]630624 name = "bytes"631625 version = "1.12.1"
@@ -2931,20 +2925,6 @@ dependencies = [
29312925 "xmltree",29322926 ]29332927 2934-[[package]]2935-name = "image"2936-version = "0.25.10"2937-source = "registry+https://github.com/rust-lang/crates.io-index"2938-checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"2939-dependencies = [2940- "bytemuck",2941- "byteorder-lite",2942- "moxcms",2943- "num-traits",2944- "zune-core",2945- "zune-jpeg",2946-]2947-29482928 [[package]]29492929 name = "indexmap"29502930 version = "1.9.3"
@@ -3687,16 +3667,6 @@ dependencies = [
36873667 "web-transport-trait",36883668 ]36893669 3690-[[package]]3691-name = "moxcms"3692-version = "0.8.1"3693-source = "registry+https://github.com/rust-lang/crates.io-index"3694-checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b"3695-dependencies = [3696- "num-traits",3697- "pxfm",3698-]3699-37003670 [[package]]37013671 name = "muda"37023672 version = "0.19.3"
@@ -3778,15 +3748,6 @@ dependencies = [
37783748 "n0-future",37793749 ]37803750 3781-[[package]]3782-name = "nasm-rs"3783-version = "0.3.2"3784-source = "registry+https://github.com/rust-lang/crates.io-index"3785-checksum = "706bf8a5e8c8ddb99128c3291d31bd21f4bcde17f0f4c20ec678d85c74faa149"3786-dependencies = [3787- "log",3788-]3789-37903751 [[package]]37913752 name = "ndk"37923753 version = "0.9.0"
@@ -4415,27 +4376,6 @@ version = "0.3.1"
44154376 source = "registry+https://github.com/rust-lang/crates.io-index"44164377 checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"44174378 4418-[[package]]4419-name = "openh264"4420-version = "0.7.2"4421-source = "registry+https://github.com/rust-lang/crates.io-index"4422-checksum = "4e0b96c5808e7e5cf977056b4da08796895f993570b4a0ad658f90764c3c18fe"4423-dependencies = [4424- "openh264-sys2",4425- "wide",4426-]4427-4428-[[package]]4429-name = "openh264-sys2"4430-version = "0.7.1"4431-source = "registry+https://github.com/rust-lang/crates.io-index"4432-checksum = "881ad6de62993ff97a2cfbc63e5f127ecc930255718f9adc4c68c1dfeb308132"4433-dependencies = [4434- "cc",4435- "nasm-rs",4436- "walkdir",4437-]4438-44394379 [[package]]44404380 name = "openssl-probe"44414381 version = "0.2.1"
@@ -4778,19 +4718,16 @@ name = "portal"
47784718 version = "0.1.0"47794719 dependencies = [47804720 "anyhow",4781- "base64 0.22.1",47824721 "bytes",47834722 "ctap-hid-fido2",47844723 "ed25519-dalek",47854724 "getrandom 0.4.3",47864725 "hang",4787- "image",47884726 "iroh",47894727 "iroh-moq",47904728 "libc",47914729 "log",47924730 "moq-net",4793- "openh264",47944731 "serde",47954732 "serde_json",47964733 "sha2 0.10.9",
@@ -4954,12 +4891,6 @@ dependencies = [
49544891 "unicode-ident",49554892 ]49564893 4957-[[package]]4958-name = "pxfm"4959-version = "0.1.30"4960-source = "registry+https://github.com/rust-lang/crates.io-index"4961-checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea"4962-49634894 [[package]]49644895 name = "quick-xml"49654896 version = "0.41.0"
@@ -5309,15 +5240,6 @@ version = "1.0.23"
53095240 source = "registry+https://github.com/rust-lang/crates.io-index"53105241 checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"53115242 5312-[[package]]5313-name = "safe_arch"5314-version = "0.7.4"5315-source = "registry+https://github.com/rust-lang/crates.io-index"5316-checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323"5317-dependencies = [5318- "bytemuck",5319-]5320-53215243 [[package]]53225244 name = "same-file"53235245 version = "1.0.6"
@@ -7465,16 +7387,6 @@ dependencies = [
74657387 "windows-core 0.61.2",74667388 ]74677389 7468-[[package]]7469-name = "wide"7470-version = "0.7.33"7471-source = "registry+https://github.com/rust-lang/crates.io-index"7472-checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03"7473-dependencies = [7474- "bytemuck",7475- "safe_arch",7476-]7477-74787390 [[package]]74797391 name = "widestring"74807392 version = "1.2.1"
@@ -8366,21 +8278,6 @@ version = "1.0.23"
83668278 source = "registry+https://github.com/rust-lang/crates.io-index"83678279 checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"83688280 8369-[[package]]8370-name = "zune-core"8371-version = "0.5.1"8372-source = "registry+https://github.com/rust-lang/crates.io-index"8373-checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9"8374-8375-[[package]]8376-name = "zune-jpeg"8377-version = "0.5.15"8378-source = "registry+https://github.com/rust-lang/crates.io-index"8379-checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"8380-dependencies = [8381- "zune-core",8382-]8383-83848281 [[package]]83858282 name = "zvariant"83868283 version = "5.13.1"
M crates/sigil-host/src/bin/sigil-probe.rs +20 -520
@@ -17,12 +17,11 @@ use sigil_protocol::{
1717 CONTROL_ALPN_V1, Capability, ClientHello, FrameFlags, GAMEPAD_AXIS_MAX, GAMEPAD_AXIS_MIN,1818 GAMEPAD_TRIGGER_MAX, GamepadState, INPUT_ALPN_V1, INVITATION_CLOCK_SKEW_SECS, InputAck,1919 InputEvent, InvitationGrants, KeyframeRequestReasonV3, MAX_INVITATION_TOKEN_LEN,20- MAX_MEDIA_GROUP_BYTES_V3, MAX_MEDIA_OBJECT_ID_V3, MEDIA_ALPN_V1, MEDIA_ALPN_V2, MEDIA_ALPN_V3,21- MEDIA_FEEDBACK_ALPN_V1, MediaCodec, MediaControlRequestV3, MediaFeedbackFlags,22- MediaFeedbackReportV1, MediaFrame, MediaObjectV3, PointerPosition, PointerSurfaceDimensions,23- ProtocolError, SignedInvitation, decode_media_frame_object, media_moq_broadcast_name,24- read_adaptive_bitrate_decision_v1, read_host_hello, read_input_ack, read_media_frame,25- read_media_object, read_media_object_v3, write_client_hello, write_input_event,20+ MAX_MEDIA_GROUP_BYTES_V3, MAX_MEDIA_OBJECT_ID_V3, MEDIA_ALPN_V3, MEDIA_FEEDBACK_ALPN_V1,21+ MediaCodec, MediaControlRequestV3, MediaFeedbackFlags, MediaFeedbackReportV1, MediaFrame,22+ MediaObjectV3, PointerPosition, PointerSurfaceDimensions, ProtocolError, SignedInvitation,23+ decode_media_frame_object, media_moq_broadcast_name, read_adaptive_bitrate_decision_v1,24+ read_host_hello, read_input_ack, read_media_object_v3, write_client_hello, write_input_event,2625 write_media_control_request_v3, write_media_feedback_report_v1,2726 };2827
@@ -69,16 +68,8 @@ struct Args {
6968 expect_size: Option<(u16, u16)>,7069 /// Exercise custom grouped v3 media instead of upstream MoQ. Intended only7170 /// for compatibility validation.72- #[arg(long, conflicts_with_all = ["media_v2", "media_v1"])]71+ #[arg(long)]7372 media_v3: bool,74- /// Exercise independent v2 media objects instead of upstream MoQ.75- /// Intended only for compatibility validation.76- #[arg(long, conflicts_with_all = ["media_v3", "media_v1"])]77- media_v2: bool,78- /// Exercise the reliable ordered v1 media stream instead of upstream MoQ.79- /// Intended only for compatibility validation.80- #[arg(long, conflicts_with_all = ["media_v3", "media_v2"])]81- media_v1: bool,8273 /// Request a configured recovery keyframe after three accepted frames,8374 /// then prove no delta history is delivered before the recovery barrier.8475 #[arg(long)]
@@ -92,7 +83,7 @@ struct Args {
9283 #[arg(9384 long,9485 value_parser = clap::value_parser!(u64).range(250..=5_000),95- conflicts_with_all = ["media_v3", "media_v2", "media_v1"]86+ conflicts_with = "media_v3"9687 )]9788 slow_consumer_ms: Option<u64>,9889 /// Require gamepad negotiation and emit one bounded non-neutral snapshot
@@ -112,7 +103,7 @@ struct Args {
112103 /// validating shadow-only encoder configurations.113104 #[arg(114105 long,115- conflicts_with_all = ["media_v3", "media_v2", "media_v1", "resolution_stall_smoke"]106+ conflicts_with_all = ["media_v3", "resolution_stall_smoke"]116107 )]117108 feedback_smoke: bool,118109 /// Prove adaptive-resolution liveness across a controlled Gamescope stall.
@@ -122,8 +113,6 @@ struct Args {
122113 value_name = "GATE_DIR",123114 conflicts_with_all = [124115 "media_v3",125- "media_v2",126- "media_v1",127116 "keyframe_smoke",128117 "slow_consumer_ms",129118 "minimum_fps",
@@ -137,8 +126,6 @@ struct Args {
137126 enum MediaTransport {138127 UpstreamMoq,139128 GroupedV3,140- IndependentV2,141- ReliableV1,142129 }143130 144131 impl MediaTransport {
@@ -146,8 +133,6 @@ impl MediaTransport {
146133 match self {147134 Self::UpstreamMoq => CONTROL_ALPN_V1,148135 Self::GroupedV3 => MEDIA_ALPN_V3,149- Self::IndependentV2 => MEDIA_ALPN_V2,150- Self::ReliableV1 => MEDIA_ALPN_V1,151136 }152137 }153138
@@ -155,8 +140,6 @@ impl MediaTransport {
155140 match self {156141 Self::UpstreamMoq => "iroh-moq",157142 Self::GroupedV3 => "grouped-v3",158- Self::IndependentV2 => "independent-v2",159- Self::ReliableV1 => "reliable-v1",160143 }161144 }162145
@@ -165,8 +148,6 @@ impl MediaTransport {
165148 Self::UpstreamMoq => std::str::from_utf8(iroh_moq::ALPN)166149 .expect("pinned iroh-moq ALPN must be printable UTF-8"),167150 Self::GroupedV3 => "sigil/media/3",168- Self::IndependentV2 => "sigil/media/2",169- Self::ReliableV1 => "sigil/media/1",170151 }171152 }172153 }
@@ -182,167 +163,6 @@ const RESOLUTION_STALL_RECOVERY_TIMEOUT: Duration = Duration::from_secs(2);
182163 const RESOLUTION_STALL_GATE_POLL_INTERVAL: Duration = Duration::from_millis(50);183164 const MAX_INVITATION_FILE_BYTES: u64 = (MAX_INVITATION_TOKEN_LEN + 2) as u64;184165 185-#[derive(Debug)]186-enum MediaObjectOutcome {187- Frame { index: u64, frame: MediaFrame },188- Dropped { index: u64 },189- Malformed { index: u64, error: ProtocolError },190-}191-192-impl MediaObjectOutcome {193- fn index(&self) -> u64 {194- match self {195- Self::Frame { index, .. } | Self::Dropped { index } | Self::Malformed { index, .. } => {196- *index197- }198- }199- }200-201- fn is_fast_forward_barrier(&self) -> bool {202- let Self::Frame { frame, .. } = self else {203- return false;204- };205- frame.header.flags.contains(FrameFlags::KEYFRAME)206- && frame.header.flags.contains(FrameFlags::CODEC_CONFIG)207- && frame.header.flags.contains(FrameFlags::DISCONTINUITY)208- }209-}210-211-#[derive(Debug)]212-struct MediaObjectReorder {213- next_index: u64,214- completed: BTreeMap<u64, MediaObjectOutcome>,215-}216-217-impl MediaObjectReorder {218- fn new(first_index: u64) -> Self {219- Self {220- next_index: first_index,221- completed: BTreeMap::new(),222- }223- }224-225- fn pending_len(&self) -> usize {226- self.completed.len()227- }228-229- fn push(&mut self, outcome: MediaObjectOutcome) -> Result<Option<MediaObjectOutcome>> {230- if matches!(outcome, MediaObjectOutcome::Malformed { .. }) {231- // Malformed objects remain terminal as soon as their read completes.232- return Ok(Some(outcome));233- }234- let index = outcome.index();235- if index < self.next_index {236- return Ok(Some(outcome));237- }238- if outcome.is_fast_forward_barrier() {239- self.completed240- .retain(|completed_index, _| *completed_index > index);241- self.next_index = index242- .checked_add(1)243- .context("media object reorder overflowed")?;244- return Ok(Some(outcome));245- }246- ensure!(247- self.completed.insert(index, outcome).is_none(),248- "media object {index} completed more than once"249- );250- self.take_next()251- }252-253- fn take_next(&mut self) -> Result<Option<MediaObjectOutcome>> {254- let Some(outcome) = self.completed.remove(&self.next_index) else {255- return Ok(None);256- };257- self.next_index = self258- .next_index259- .checked_add(1)260- .context("media object reorder overflowed")?;261- Ok(Some(outcome))262- }263-}264-265-struct MediaObjectReceiver {266- connection: iroh::endpoint::Connection,267- reads: tokio::task::JoinSet<MediaObjectOutcome>,268- reorder: MediaObjectReorder,269- next_index: u64,270- accepting: bool,271- read_timeout: Duration,272-}273-274-impl MediaObjectReceiver {275- fn new(connection: iroh::endpoint::Connection, read_timeout: Duration) -> Self {276- Self {277- connection,278- reads: tokio::task::JoinSet::new(),279- reorder: MediaObjectReorder::new(0),280- next_index: 0,281- accepting: true,282- read_timeout,283- }284- }285-286- async fn next(&mut self) -> Result<Option<MediaObjectOutcome>> {287- loop {288- if let Some(completed) = self.reorder.take_next()? {289- return Ok(Some(completed));290- }291- if !self.accepting && self.reads.is_empty() {292- ensure!(293- self.reorder.pending_len() == 0,294- "media connection closed with an incomplete object order"295- );296- return Ok(None);297- }298-299- tokio::select! {300- accepted = self.connection.accept_uni(),301- if self.accepting302- && self.reads.len() + self.reorder.pending_len()303- < MEDIA_OBJECT_CAPACITY => {304- match accepted {305- Ok(mut stream) => {306- let index = self.next_index;307- self.next_index = self308- .next_index309- .checked_add(1)310- .context("media object index overflowed")?;311- let read_timeout = self.read_timeout;312- self.reads.spawn(async move {313- match tokio::time::timeout(read_timeout, read_media_object(&mut stream)).await {314- Ok(Ok(frame)) => MediaObjectOutcome::Frame { index, frame },315- Ok(Err(ProtocolError::Io(_))) => MediaObjectOutcome::Dropped { index },316- Ok(Err(error)) => MediaObjectOutcome::Malformed { index, error },317- Err(_) => MediaObjectOutcome::Dropped { index },318- }319- });320- }321- Err(_) => self.accepting = false,322- }323- }324- completed = self.reads.join_next(), if !self.reads.is_empty() => {325- match completed.expect("guarded by non-empty media object task set") {326- Ok(outcome) => {327- if let Some(completed) = self.reorder.push(outcome)? {328- return Ok(Some(completed));329- }330- }331- Err(error) if error.is_cancelled() => continue,332- Err(error) => return Err(error).context("media object read task failed"),333- }334- }335- }336- }337- }338-}339-340-impl Drop for MediaObjectReceiver {341- fn drop(&mut self) {342- self.reads.abort_all();343- }344-}345-346166 #[derive(Debug)]347167 enum MediaObjectOutcomeV3 {348168 Object {
@@ -524,71 +344,6 @@ impl Drop for MediaObjectReceiverV3 {
524344 }525345 }526346 527-#[derive(Clone, Copy, Debug, PartialEq, Eq)]528-enum MediaObjectDecision {529- Deliver,530- DropLate,531- DropUntilKeyframe,532-}533-534-#[derive(Debug, Default)]535-struct MediaObjectSequence {536- completion_watermark: Option<u64>,537- last_sequence: Option<u64>,538- waiting_for_keyframe: bool,539-}540-541-impl MediaObjectSequence {542- fn new() -> Self {543- Self {544- waiting_for_keyframe: true,545- ..Self::default()546- }547- }548-549- fn note_drop(&mut self, index: u64) -> bool {550- if self551- .completion_watermark552- .is_some_and(|watermark| index <= watermark)553- {554- return false;555- }556- self.completion_watermark = Some(index);557- self.waiting_for_keyframe = true;558- true559- }560-561- fn classify(&mut self, index: u64, frame: &MediaFrame) -> MediaObjectDecision {562- if self563- .completion_watermark564- .is_some_and(|watermark| index <= watermark)565- {566- return MediaObjectDecision::DropLate;567- }568- self.completion_watermark = Some(index);569-570- let independently_decodable = frame.header.flags.contains(FrameFlags::KEYFRAME)571- && frame.header.flags.contains(FrameFlags::CODEC_CONFIG);572- let sequence_contiguous = self573- .last_sequence574- .is_none_or(|last| last.checked_add(1) == Some(frame.header.sequence));575- let sequence_monotonic = self576- .last_sequence577- .is_none_or(|last| frame.header.sequence > last);578-579- let resync_required =580- !sequence_monotonic || self.waiting_for_keyframe || !sequence_contiguous;581- if resync_required && (!independently_decodable || !sequence_monotonic) {582- self.waiting_for_keyframe = true;583- return MediaObjectDecision::DropUntilKeyframe;584- }585-586- self.last_sequence = Some(frame.header.sequence);587- self.waiting_for_keyframe = false;588- MediaObjectDecision::Deliver589- }590-}591-592347 #[derive(Clone, Copy, Debug, PartialEq, Eq)]593348 enum MediaObjectDecisionV3 {594349 Deliver { discontinuity: bool },
@@ -985,20 +740,6 @@ struct AcceptedMedia {
985740 v3_group_id: Option<u64>,986741 }987742 988-impl From<MediaFrame> for AcceptedMedia {989- fn from(frame: MediaFrame) -> Self {990- Self {991- flags: frame.header.flags,992- width: frame.header.width,993- height: frame.header.height,994- sequence: frame.header.sequence,995- capture_timestamp_us: frame.header.capture_timestamp_us,996- payload_len: frame.payload.len(),997- v3_group_id: None,998- }999- }1000-}1001-1002743 impl From<MediaObjectV3> for AcceptedMedia {1003744 fn from(object: MediaObjectV3) -> Self {1004745 Self {
@@ -1465,10 +1206,6 @@ async fn main() -> Result<()> {
14651206 args.timeout_seconds > 0,14661207 "--timeout-seconds must be greater than zero"14671208 );1468- ensure!(1469- !args.keyframe_smoke || (!args.media_v1 && !args.media_v2),1470- "--keyframe-smoke requires upstream MoQ or grouped v3 media"1471- );14721209 ensure!(14731210 !args.keyframe_smoke || args.frames >= KEYFRAME_SMOKE_MINIMUM_FRAMES,14741211 "--keyframe-smoke requires at least {KEYFRAME_SMOKE_MINIMUM_FRAMES} accepted frames"
@@ -1506,11 +1243,7 @@ async fn main() -> Result<()> {
15061243 .context("binding probe endpoint")?;15071244 let _ = tokio::time::timeout(Duration::from_secs(10), endpoint.online()).await;15081245 let address = EndpointAddr::new(args.node_id);1509- let media_transport = if args.media_v1 {1510- MediaTransport::ReliableV11511- } else if args.media_v2 {1512- MediaTransport::IndependentV21513- } else if args.media_v3 {1246+ let media_transport = if args.media_v3 {15141247 MediaTransport::GroupedV315151248 } else {15161249 MediaTransport::UpstreamMoq
@@ -1540,24 +1273,9 @@ async fn main() -> Result<()> {
15401273 )15411274 .await?;15421275 let session_id = media_negotiation.session_id;1543- let mut media_send = Some(media_send);1544- let mut media_recv = Some(media_recv);1545- match media_transport {1546- MediaTransport::UpstreamMoq | MediaTransport::GroupedV3 => {1547- // Sigil finishes its response half after HostHello. Keep our send1548- // half alive as the bounded keyframe-control stream.1549- drop(media_recv.take());1550- }1551- MediaTransport::IndependentV2 => {1552- media_send1553- .take()1554- .expect("media handshake stream is present")1555- .finish()1556- .context("finishing media v2 handshake request")?;1557- drop(media_recv.take());1558- }1559- MediaTransport::ReliableV1 => {}1560- }1276+ // Sigil finishes its response half after HostHello. Keep our send half1277+ // alive as the bounded keyframe-control stream.1278+ drop(media_recv);15611279 15621280 let mut moq_receiver = if media_transport == MediaTransport::UpstreamMoq {15631281 Some(
@@ -1780,11 +1498,9 @@ async fn main() -> Result<()> {
17801498 .finish()17811499 .context("finishing media feedback stream")?;17821500 input_send.finish().context("finishing input stream")?;1783- if let Some(mut media_send) = media_send {1784- media_send1785- .finish()1786- .context("finishing media request stream")?;1787- }1501+ media_send1502+ .finish()1503+ .context("finishing media request stream")?;17881504 feedback_connection.close(0_u32.into(), b"probe complete");17891505 input_connection.close(0_u32.into(), b"probe complete");17901506 moq_receiver
@@ -1844,13 +1560,6 @@ async fn main() -> Result<()> {
18441560 let mut dimensions = None;18451561 let mut first_accepted_at = None;18461562 let mut last_accepted_at = None;1847- let mut object_receiver = (media_transport == MediaTransport::IndependentV2).then(|| {1848- MediaObjectReceiver::new(1849- media_connection.clone(),1850- Duration::from_secs(args.timeout_seconds),1851- )1852- });1853- let mut object_sequence = MediaObjectSequence::new();18541563 let mut object_receiver_v3 = (media_transport == MediaTransport::GroupedV3).then(|| {18551564 MediaObjectReceiverV3::new(18561565 media_connection.clone(),
@@ -2029,57 +1738,6 @@ async fn main() -> Result<()> {
20291738 }20301739 }20311740 },2032- MediaTransport::ReliableV1 => (2033- tokio::time::timeout(2034- Duration::from_secs(args.timeout_seconds),2035- read_media_frame(2036- media_recv2037- .as_mut()2038- .expect("v1 media receive stream is present"),2039- ),2040- )2041- .await2042- .context("timed out waiting for media frame")??2043- .context("host closed the media stream")?2044- .into(),2045- false,2046- ),2047- MediaTransport::IndependentV2 => loop {2048- let outcome = tokio::time::timeout(2049- Duration::from_secs(args.timeout_seconds)2050- .saturating_add(Duration::from_secs(1)),2051- object_receiver2052- .as_mut()2053- .expect("v2 media object receiver is present")2054- .next(),2055- )2056- .await2057- .context("timed out waiting for media object")??2058- .context("host closed the media object connection")?;2059- match outcome {2060- MediaObjectOutcome::Dropped { index } => {2061- if object_sequence.note_drop(index) {2062- media_objects_dropped = media_objects_dropped.saturating_add(1);2063- } else {2064- media_objects_late = media_objects_late.saturating_add(1);2065- }2066- }2067- MediaObjectOutcome::Malformed { index, error } => {2068- bail!("media object {index} is malformed: {error}");2069- }2070- MediaObjectOutcome::Frame { index, frame } => {2071- match object_sequence.classify(index, &frame) {2072- MediaObjectDecision::Deliver => break (frame.into(), false),2073- MediaObjectDecision::DropLate => {2074- media_objects_late = media_objects_late.saturating_add(1);2075- }2076- MediaObjectDecision::DropUntilKeyframe => {2077- media_objects_dropped = media_objects_dropped.saturating_add(1);2078- }2079- }2080- }2081- }2082- },20831741 MediaTransport::GroupedV3 => loop {20841742 let outcome = tokio::time::timeout(20851743 Duration::from_secs(args.timeout_seconds)
@@ -2235,12 +1893,7 @@ async fn main() -> Result<()> {
22351893 );22361894 tokio::time::timeout(22371895 Duration::from_secs(1),2238- write_media_control_request_v3(2239- media_send2240- .as_mut()2241- .expect("v3 media control stream is present"),2242- &request,2243- ),1896+ write_media_control_request_v3(&mut media_send, &request),22441897 )22451898 .await22461899 .context("timed out writing v3 keyframe request")??;
@@ -2260,9 +1913,6 @@ async fn main() -> Result<()> {
22601913 );22611914 }22621915 MediaTransport::GroupedV3 => object_sequence_v3.request_recovery(),2263- MediaTransport::IndependentV2 | MediaTransport::ReliableV1 => {2264- unreachable!("keyframe smoke excludes v2 and v1")2265- }22661916 }22671917 }22681918 }
@@ -2355,11 +2005,9 @@ async fn main() -> Result<()> {
23552005 )23562006 });23572007 input_send.finish().context("finishing input stream")?;2358- if let Some(mut media_send) = media_send {2359- media_send2360- .finish()2361- .context("finishing media request stream")?;2362- }2008+ media_send2009+ .finish()2010+ .context("finishing media request stream")?;23632011 // Close both protocol connections explicitly. Relying only on endpoint23642012 // teardown can leave a peer waiting for QUIC idle timeout during repeated23652013 // short-lived probes, which obscures whether the host released its
@@ -3167,13 +2815,6 @@ mod tests {
31672815 .unwrap()31682816 }31692817 3170- fn media_outcome(index: u64, sequence: u64, flags: FrameFlags) -> MediaObjectOutcome {3171- MediaObjectOutcome::Frame {3172- index,3173- frame: media_frame(sequence, flags),3174- }3175- }3176-31772818 fn media_object_v3(31782819 group_id: u64,31792820 object_id: u32,
@@ -3211,62 +2852,6 @@ mod tests {
32112852 }32122853 }32132854 3214- #[test]3215- fn media_object_reorder_restores_accept_order_before_sequence_checks() {3216- let keyframe = FrameFlags::KEYFRAME.union(FrameFlags::CODEC_CONFIG);3217- let mut reorder = MediaObjectReorder::new(0);3218-3219- assert!(3220- reorder3221- .push(media_outcome(1, 11, FrameFlags::NONE))3222- .unwrap()3223- .is_none()3224- );3225- assert_eq!(reorder.pending_len(), 1);3226- assert_eq!(3227- reorder3228- .push(media_outcome(0, 10, keyframe))3229- .unwrap()3230- .unwrap()3231- .index(),3232- 03233- );3234- assert_eq!(reorder.take_next().unwrap().unwrap().index(), 1);3235- assert_eq!(reorder.pending_len(), 0);3236- }3237-3238- #[test]3239- fn discontinuity_keyframe_is_an_explicit_latest_frame_barrier() {3240- let barrier = FrameFlags::KEYFRAME3241- .union(FrameFlags::CODEC_CONFIG)3242- .union(FrameFlags::DISCONTINUITY);3243- let mut reorder = MediaObjectReorder::new(0);3244-3245- assert!(3246- reorder3247- .push(media_outcome(1, 11, FrameFlags::NONE))3248- .unwrap()3249- .is_none()3250- );3251- assert_eq!(3252- reorder3253- .push(media_outcome(2, 20, barrier))3254- .unwrap()3255- .unwrap()3256- .index(),3257- 23258- );3259- assert_eq!(reorder.pending_len(), 0);3260- assert_eq!(3261- reorder3262- .push(media_outcome(0, 10, FrameFlags::NONE))3263- .unwrap()3264- .unwrap()3265- .index(),3266- 03267- );3268- }3269-32702855 #[test]32712856 fn media_v3_discontinuity_object_zero_fast_forwards_accept_order() {32722857 let keyframe = FrameFlags::KEYFRAME.union(FrameFlags::CODEC_CONFIG);
@@ -3331,79 +2916,6 @@ mod tests {
33312916 assert!(accepted_frame_rate(55, Duration::from_secs(1)).unwrap() < 55.0);33322917 }33332918 3334- #[test]3335- fn media_objects_wait_for_a_decodable_keyframe_then_deliver_contiguous_frames() {3336- let mut sequence = MediaObjectSequence::new();3337- let delta = media_frame(1, FrameFlags::NONE);3338- let keyframe = media_frame(2, FrameFlags::KEYFRAME.union(FrameFlags::CODEC_CONFIG));3339- let next_delta = media_frame(3, FrameFlags::NONE);3340-3341- assert_eq!(3342- sequence.classify(0, &delta),3343- MediaObjectDecision::DropUntilKeyframe3344- );3345- assert_eq!(3346- sequence.classify(1, &keyframe),3347- MediaObjectDecision::Deliver3348- );3349- assert_eq!(3350- sequence.classify(2, &next_delta),3351- MediaObjectDecision::Deliver3352- );3353- }3354-3355- #[test]3356- fn dropped_objects_force_resync_and_late_completions_cannot_rewind() {3357- let mut sequence = MediaObjectSequence::new();3358- let first_keyframe = media_frame(10, FrameFlags::KEYFRAME.union(FrameFlags::CODEC_CONFIG));3359- let stale_delta = media_frame(11, FrameFlags::NONE);3360- let replacement_keyframe =3361- media_frame(20, FrameFlags::KEYFRAME.union(FrameFlags::CODEC_CONFIG));3362-3363- assert_eq!(3364- sequence.classify(0, &first_keyframe),3365- MediaObjectDecision::Deliver3366- );3367- assert!(sequence.note_drop(1));3368- assert_eq!(3369- sequence.classify(2, &replacement_keyframe),3370- MediaObjectDecision::Deliver3371- );3372- assert_eq!(3373- sequence.classify(1, &stale_delta),3374- MediaObjectDecision::DropLate3375- );3376- assert!(!sequence.note_drop(1));3377-3378- let next_delta = media_frame(21, FrameFlags::NONE);3379- assert_eq!(3380- sequence.classify(3, &next_delta),3381- MediaObjectDecision::Deliver3382- );3383- }3384-3385- #[test]3386- fn media_sequence_gap_drops_deltas_until_replacement_keyframe() {3387- let mut sequence = MediaObjectSequence::new();3388- let keyframe = media_frame(40, FrameFlags::KEYFRAME.union(FrameFlags::CODEC_CONFIG));3389- let gap_delta = media_frame(42, FrameFlags::NONE);3390- let replacement_keyframe =3391- media_frame(50, FrameFlags::KEYFRAME.union(FrameFlags::CODEC_CONFIG));3392-3393- assert_eq!(3394- sequence.classify(0, &keyframe),3395- MediaObjectDecision::Deliver3396- );3397- assert_eq!(3398- sequence.classify(1, &gap_delta),3399- MediaObjectDecision::DropUntilKeyframe3400- );3401- assert_eq!(3402- sequence.classify(2, &replacement_keyframe),3403- MediaObjectDecision::Deliver3404- );3405- }3406-34072919 #[test]34082920 fn media_v3_enforces_wire_group_object_and_sequence_continuity() {34092921 let keyframe = FrameFlags::KEYFRAME.union(FrameFlags::CODEC_CONFIG);
@@ -3810,11 +3322,9 @@ mod tests {
38103322 }38113323 38123324 #[test]3813- fn media_compatibility_flags_are_mutually_exclusive() {3325+ fn media_compatibility_flag_defaults_and_conflicts_are_enforced() {38143326 let node_id = SecretKey::from_bytes(&[0x2a; 32]).public().to_string();38153327 let default = Args::try_parse_from(["sigil-probe", "--node-id", &node_id]).unwrap();3816- assert!(!default.media_v1);3817- assert!(!default.media_v2);38183328 assert!(!default.media_v3);38193329 assert_eq!(default.identity, None);38203330 assert_eq!(default.invitation, None);
@@ -3833,16 +3343,6 @@ mod tests {
38333343 Args::try_parse_from(["sigil-probe", "--node-id", &node_id, "--feedback-smoke"])38343344 .unwrap();38353345 assert!(feedback.feedback_smoke);3836- for flags in [3837- ["--media-v1", "--media-v2"],3838- ["--media-v1", "--media-v3"],3839- ["--media-v2", "--media-v3"],3840- ] {3841- assert!(3842- Args::try_parse_from(["sigil-probe", "--node-id", &node_id, flags[0], flags[1],])3843- .is_err()3844- );3845- }38463346 assert!(38473347 Args::try_parse_from([38483348 "sigil-probe",
M crates/sigil-host/src/main.rs +2 -18
M crates/sigil-host/src/server/handlers.rs +0 -48
@@ -50,20 +50,6 @@ where
5050 }5151 }5252 53-#[derive(Clone, Debug)]54-pub struct MediaHandler {55- pub config: HostConfig,56- pub sessions: Arc<SessionRegistry>,57- pub authorization: AuthorizationPolicy,58-}59-60-#[derive(Clone, Debug)]61-pub struct MediaV2Handler {62- pub config: HostConfig,63- pub sessions: Arc<SessionRegistry>,64- pub authorization: AuthorizationPolicy,65-}66-6753 #[derive(Clone, Debug)]6854 pub struct MediaV3Handler {6955 pub config: HostConfig,
@@ -97,40 +83,6 @@ pub struct AuthorizedMoqHandler {
9783 pub origin: Origin,9884 }9985 100-impl ProtocolHandler for MediaHandler {101- async fn accept(&self, connection: Connection) -> Result<(), iroh::protocol::AcceptError> {102- let remote = connection.remote_id();103- if let Err(error) = serve_media(104- connection,105- self.config.clone(),106- &self.sessions,107- &self.authorization,108- )109- .await110- {111- warn!(%remote, %error, "media connection ended");112- }113- Ok(())114- }115-}116-117-impl ProtocolHandler for MediaV2Handler {118- async fn accept(&self, connection: Connection) -> Result<(), iroh::protocol::AcceptError> {119- let remote = connection.remote_id();120- if let Err(error) = serve_media_v2(121- connection,122- self.config.clone(),123- &self.sessions,124- &self.authorization,125- )126- .await127- {128- warn!(%remote, %error, "media v2 connection ended");129- }130- Ok(())131- }132-}133-13486 impl ProtocolHandler for MediaV3Handler {13587 async fn accept(&self, connection: Connection) -> Result<(), iroh::protocol::AcceptError> {13688 let remote = connection.remote_id();
D crates/sigil-host/src/server/media_v2.rs +0 -821
@@ -1,821 +0,0 @@
1-use super::*;2-3-const MEDIA_WRITE_TIMEOUT: Duration = Duration::from_secs(5);4-const MEDIA_V2_PEER_ACK_TIMEOUT: Duration = Duration::from_secs(2);5-const MEDIA_V2_IN_FLIGHT_CAPACITY: usize = 4;6-const MEDIA_V2_KEYFRAME_PRIORITY: i32 = 10;7-const MEDIA_V2_DELTA_PRIORITY: i32 = 0;8-const MEDIA_V2_RESET_CODE: u32 = 0x5356;9-10-#[derive(Clone, Copy, Debug, PartialEq, Eq)]11-enum MediaActivity {12- SourceChanged,13- PeerDisconnected,14-}15-16-#[derive(Debug, PartialEq, Eq)]17-enum MediaV2ScheduleDecision {18- Send {19- discontinuity: bool,20- cancel_sequences: Vec<u64>,21- },22- SkipUntilKeyframe,23- EnterResync {24- cancel_sequences: Vec<u64>,25- },26-}27-28-#[derive(Debug)]29-struct MediaV2Scheduler {30- in_flight: Vec<u64>,31- last_scheduled_sequence: Option<u64>,32- waiting_for_keyframe: bool,33- discontinuity_pending: bool,34-}35-36-impl Default for MediaV2Scheduler {37- fn default() -> Self {38- Self {39- in_flight: Vec::with_capacity(MEDIA_V2_IN_FLIGHT_CAPACITY),40- last_scheduled_sequence: None,41- waiting_for_keyframe: true,42- discontinuity_pending: false,43- }44- }45-}46-47-impl MediaV2Scheduler {48- fn schedule(49- &mut self,50- sequence: u64,51- independently_decodable: bool,52- ) -> MediaV2ScheduleDecision {53- if self54- .last_scheduled_sequence55- .is_some_and(|last| sequence <= last)56- {57- return MediaV2ScheduleDecision::SkipUntilKeyframe;58- }59-60- let sequence_discontinuity = self61- .last_scheduled_sequence62- .is_some_and(|last| last.checked_add(1) != Some(sequence));63- if independently_decodable {64- let cancel_sequences = std::mem::take(&mut self.in_flight);65- let discontinuity = self.discontinuity_pending66- || sequence_discontinuity67- || !cancel_sequences.is_empty();68- self.in_flight.push(sequence);69- self.last_scheduled_sequence = Some(sequence);70- self.waiting_for_keyframe = false;71- self.discontinuity_pending = false;72- return MediaV2ScheduleDecision::Send {73- discontinuity,74- cancel_sequences,75- };76- }77-78- if self.waiting_for_keyframe {79- return MediaV2ScheduleDecision::SkipUntilKeyframe;80- }81- if sequence_discontinuity || self.in_flight.len() == MEDIA_V2_IN_FLIGHT_CAPACITY {82- return MediaV2ScheduleDecision::EnterResync {83- cancel_sequences: self.enter_resync(),84- };85- }86-87- self.in_flight.push(sequence);88- self.last_scheduled_sequence = Some(sequence);89- MediaV2ScheduleDecision::Send {90- discontinuity: false,91- cancel_sequences: Vec::new(),92- }93- }94-95- fn complete(&mut self, sequence: u64) {96- if let Some(index) = self.in_flight.iter().position(|value| *value == sequence) {97- self.in_flight.swap_remove(index);98- }99- }100-101- fn fail(&mut self, sequence: u64) -> Vec<u64> {102- let Some(index) = self.in_flight.iter().position(|value| *value == sequence) else {103- // A completion from an already-cancelled GOP must not poison the104- // newer GOP which replaced it.105- return Vec::new();106- };107- self.in_flight.swap_remove(index);108- self.enter_resync()109- }110-111- fn fail_all(&mut self) -> Vec<u64> {112- self.enter_resync()113- }114-115- fn enter_resync(&mut self) -> Vec<u64> {116- self.waiting_for_keyframe = true;117- self.discontinuity_pending = true;118- std::mem::take(&mut self.in_flight)119- }120-}121-122-struct ResetOnDropSendStream(Option<SendStream>);123-124-impl ResetOnDropSendStream {125- fn new(stream: SendStream) -> Self {126- Self(Some(stream))127- }128-129- fn stream(&self) -> &SendStream {130- self.0.as_ref().expect("send stream guard is armed")131- }132-133- fn stream_mut(&mut self) -> &mut SendStream {134- self.0.as_mut().expect("send stream guard is armed")135- }136-137- fn disarm(&mut self) {138- self.0.take();139- }140-}141-142-impl Drop for ResetOnDropSendStream {143- fn drop(&mut self) {144- if let Some(stream) = self.0.as_mut() {145- let _ = stream.reset(MEDIA_V2_RESET_CODE.into());146- }147- }148-}149-150-async fn wait_for_media_activity<T, F>(151- receiver: &mut tokio::sync::watch::Receiver<T>,152- peer_disconnected: Pin<&mut F>,153-) -> Result<MediaActivity>154-where155- T: Clone + Send + Sync,156- F: Future<Output = ()>,157-{158- tokio::select! {159- changed = receiver.changed() => {160- changed.context("encoded source stopped")?;161- Ok(MediaActivity::SourceChanged)162- }163- () = peer_disconnected => Ok(MediaActivity::PeerDisconnected),164- }165-}166-167-pub(super) async fn serve_media(168- connection: Connection,169- config: HostConfig,170- sessions: &Arc<SessionRegistry>,171- authorization: &AuthorizationPolicy,172-) -> Result<()> {173- let remote = connection.remote_id();174- let handshake_permit = sessions175- .pending_handshakes176- .try_acquire()177- .context("too many pending handshakes")?;178- let (mut send, mut recv) = tokio::time::timeout(HANDSHAKE_TIMEOUT, connection.accept_bi())179- .await180- .context("timed out accepting media stream")?181- .context("accepting media stream")?;182- let hello = receive_hello(&mut recv, Capability::VideoH264).await?;183- drop(handshake_permit);184- debug!(%remote, agent = %hello.agent, "media hello received");185-186- let grants = match authorization.authorize_or_redeem(187- remote,188- hello.invitation.as_deref(),189- unix_timestamp_now()?,190- ) {191- Ok(grants) => grants,192- Err(error) => {193- send_rejection(&mut send, "Portal peer is not authorized").await?;194- return Err(error.context("authorizing media peer"));195- }196- };197- ensure!(198- grants.contains(InvitationGrants::VIEW),199- "authorized media peer lacks view permission"200- );201-202- let lease = match sessions.claim(remote, hello.nonce, grants) {203- Ok(lease) => lease,204- Err(error) => {205- send_rejection(&mut send, "host already has an active client").await?;206- return Err(error);207- }208- };209-210- // `serve` has already completed the static executable and encoder211- // preflight. Resolve the live PipeWire node and create the bounded source212- // before accepting the session so plugin discovery can never sit behind an213- // accepted HostHello and the client's media-idle timeout.214- let source = match config.source {215- VideoSource::TestPattern => Ok(spawn_test_pattern(config.clone(), lease.session_clock)),216- VideoSource::GamescopePipewire => {217- let primary = spawn_gamescope_pipewire_after_static_preflight(218- config.clone(),219- lease.session_clock,220- )221- .await?;222- select_gamescope_startup_source(config.clone(), lease.session_clock, primary).await223- }224- };225- let EncodedSource {226- frames: frame_receiver,227- current_gop: mut current_gop_receiver,228- task: source_task,229- pointer_surface_dimensions,230- encoder_control,231- } = match source {232- Ok(source) => source,233- Err(error) => {234- send_rejection(&mut send, "video source is unavailable").await?;235- return Err(error);236- }237- };238- let source_task = SourceTaskGuard::new(source_task);239- sessions.install_encoder_control(remote, lease.session_id, encoder_control.clone())?;240- let _encoder_control = encoder_control;241-242- let mut media_hello = HostHello::accepted(243- lease.session_id,244- negotiated_capabilities(&hello, MEDIA_CAPABILITIES),245- );246- if let Some(dimensions) = pointer_surface_dimensions {247- media_hello = media_hello.with_pointer_surface_dimensions(dimensions);248- }249- write_host_hello(&mut send, &media_hello).await?;250- info!(%remote, session_id = lease.session_id, "media client accepted");251-252- let session_result: Result<()> = async {253- let maximum_replay_age = maximum_media_replay_age(config.framerate);254- let mut replay_cursor = MediaReplayCursor::default();255- let peer_disconnected = async {256- let result = connection.closed().await;257- debug!(%remote, ?result, "media connection closed");258- };259- tokio::pin!(peer_disconnected);260- loop {261- match wait_for_media_activity(&mut current_gop_receiver, peer_disconnected.as_mut())262- .await?263- {264- MediaActivity::PeerDisconnected => return Ok(()),265- MediaActivity::SourceChanged => {}266- }267- let Some(current_gop) = current_gop_receiver.borrow_and_update().clone() else {268- continue;269- };270- let initial_replay_started_at =271- replay_cursor.last_sequence.is_none().then(Instant::now);272- let replay_through_sequence = current_gop273- .frames274- .last()275- .map(|frame| frame.sequence)276- .context("current GOP snapshot is empty")?;277- for frame in new_current_gop_frames(current_gop, replay_cursor.last_sequence) {278- let discontinuity = match replay_cursor.classify(279- &frame,280- replay_through_sequence,281- initial_replay_started_at,282- Instant::now(),283- maximum_replay_age,284- ) {285- MediaReplayDecision::Send { discontinuity } => discontinuity,286- MediaReplayDecision::SkipUntilKeyframe => {287- debug!(288- sequence = frame.sequence,289- "waiting for keyframe with codec configuration"290- );291- continue;292- }293- MediaReplayDecision::DiscardStaleSuffix { through_sequence } => {294- debug!(295- sequence = frame.sequence,296- through_sequence,297- replay_age_micros = frame.observed_at.elapsed().as_micros(),298- maximum_replay_age_micros = maximum_replay_age.as_micros(),299- "discarding stale media suffix and waiting for keyframe"300- );301- break;302- }303- };304- let media_frame = media_frame_for_encoded(&config, &frame, discontinuity)?;305- tokio::time::timeout(306- MEDIA_WRITE_TIMEOUT,307- write_media_frame(&mut send, &media_frame),308- )309- .await310- .context("timed out writing media frame")??;311- replay_cursor.commit_sent(&frame);312- }313- }314- }315- .await;316-317- drop(current_gop_receiver);318- drop(frame_receiver);319- source_task.wait_or_abort(SOURCE_REAP_GRACE_TIMEOUT).await;320- drop(lease);321- info!(%remote, "media client released");322- session_result323-}324-325-pub(super) async fn serve_media_v2(326- connection: Connection,327- config: HostConfig,328- sessions: &Arc<SessionRegistry>,329- authorization: &AuthorizationPolicy,330-) -> Result<()> {331- let remote = connection.remote_id();332- let handshake_permit = sessions333- .pending_handshakes334- .try_acquire()335- .context("too many pending handshakes")?;336- let (mut send, mut recv) = tokio::time::timeout(HANDSHAKE_TIMEOUT, connection.accept_bi())337- .await338- .context("timed out accepting media v2 handshake stream")?339- .context("accepting media v2 handshake stream")?;340- let hello = receive_hello(&mut recv, Capability::VideoH264).await?;341- drop(handshake_permit);342- debug!(%remote, agent = %hello.agent, "media v2 hello received");343-344- let grants = match authorization.authorize_or_redeem(345- remote,346- hello.invitation.as_deref(),347- unix_timestamp_now()?,348- ) {349- Ok(grants) => grants,350- Err(error) => {351- send_rejection(&mut send, "Portal peer is not authorized").await?;352- return Err(error.context("authorizing media v2 peer"));353- }354- };355- ensure!(356- grants.contains(InvitationGrants::VIEW),357- "authorized media peer lacks view permission"358- );359-360- let lease = match sessions.claim(remote, hello.nonce, grants) {361- Ok(lease) => lease,362- Err(error) => {363- send_rejection(&mut send, "host already has an active client").await?;364- return Err(error);365- }366- };367-368- let source = match config.source {369- VideoSource::TestPattern => Ok(spawn_test_pattern(config.clone(), lease.session_clock)),370- VideoSource::GamescopePipewire => {371- let primary = spawn_gamescope_pipewire_after_static_preflight(372- config.clone(),373- lease.session_clock,374- )375- .await?;376- select_gamescope_startup_source(config.clone(), lease.session_clock, primary).await377- }378- };379- let EncodedSource {380- frames: frame_receiver,381- current_gop: mut current_gop_receiver,382- task: source_task,383- pointer_surface_dimensions,384- encoder_control,385- } = match source {386- Ok(source) => source,387- Err(error) => {388- send_rejection(&mut send, "video source is unavailable").await?;389- return Err(error);390- }391- };392- let source_task = SourceTaskGuard::new(source_task);393- sessions.install_encoder_control(remote, lease.session_id, encoder_control.clone())?;394- let _encoder_control = encoder_control;395-396- let mut media_hello = HostHello::accepted(397- lease.session_id,398- negotiated_capabilities(&hello, MEDIA_CAPABILITIES),399- );400- if let Some(dimensions) = pointer_surface_dimensions {401- media_hello = media_hello.with_pointer_surface_dimensions(dimensions);402- }403- write_host_hello(&mut send, &media_hello).await?;404- send.finish()405- .context("finishing media v2 handshake response")?;406- drop(send);407- drop(recv);408- info!(%remote, session_id = lease.session_id, "media v2 client accepted");409-410- let session_result =411- run_media_v2_session(&connection, &config, &mut current_gop_receiver, remote).await;412-413- drop(current_gop_receiver);414- drop(frame_receiver);415- source_task.wait_or_abort(SOURCE_REAP_GRACE_TIMEOUT).await;416- drop(lease);417- info!(%remote, "media v2 client released");418- session_result419-}420-421-async fn run_media_v2_session(422- connection: &Connection,423- config: &HostConfig,424- current_gop_receiver: &mut tokio::sync::watch::Receiver<Option<EncodedGop>>,425- remote: EndpointId,426-) -> Result<()> {427- let maximum_replay_age = maximum_media_replay_age(config.framerate);428- let mut replay_cursor = MediaReplayCursor::default();429- let mut scheduler = MediaV2Scheduler::default();430- let mut send_tasks = tokio::task::JoinSet::new();431-432- let result = loop {433- tokio::select! {434- closed = connection.closed() => {435- debug!(%remote, ?closed, "media v2 connection closed");436- break Ok(());437- }438- task = send_tasks.join_next(), if !send_tasks.is_empty() => {439- match task.expect("guarded by non-empty send task set") {440- Ok((sequence, Ok(()))) => scheduler.complete(sequence),441- Ok((sequence, Err(error))) => {442- warn!(sequence, %error, "media v2 object send failed; waiting for keyframe");443- if !scheduler.fail(sequence).is_empty() {444- send_tasks.abort_all();445- }446- }447- Err(error) if error.is_cancelled() => {}448- Err(error) => {449- warn!(%error, "media v2 object task failed; waiting for keyframe");450- if !scheduler.fail_all().is_empty() {451- send_tasks.abort_all();452- }453- }454- }455- }456- changed = current_gop_receiver.changed() => {457- if let Err(error) = changed {458- break Err(error).context("encoded source stopped");459- }460- let Some(current_gop) = current_gop_receiver.borrow_and_update().clone() else {461- continue;462- };463- let initial_replay_started_at =464- replay_cursor.last_sequence.is_none().then(Instant::now);465- let replay_through_sequence = current_gop466- .frames467- .last()468- .map(|frame| frame.sequence)469- .context("current GOP snapshot is empty")?;470-471- for frame in new_current_gop_frames(current_gop, replay_cursor.last_sequence) {472- let replay_discontinuity = match replay_cursor.classify(473- &frame,474- replay_through_sequence,475- initial_replay_started_at,476- Instant::now(),477- maximum_replay_age,478- ) {479- MediaReplayDecision::Send { discontinuity } => discontinuity,480- MediaReplayDecision::SkipUntilKeyframe => continue,481- MediaReplayDecision::DiscardStaleSuffix { .. } => break,482- };483- let independently_decodable = frame.keyframe && frame.codec_config;484- let (scheduler_discontinuity, cancel_sequences) =485- match scheduler.schedule(frame.sequence, independently_decodable) {486- MediaV2ScheduleDecision::Send {487- discontinuity,488- cancel_sequences,489- } => (discontinuity, cancel_sequences),490- MediaV2ScheduleDecision::SkipUntilKeyframe => continue,491- MediaV2ScheduleDecision::EnterResync { cancel_sequences } => {492- if !cancel_sequences.is_empty() {493- send_tasks.abort_all();494- }495- continue;496- }497- };498- if !cancel_sequences.is_empty() {499- debug!(500- sequence = frame.sequence,501- ?cancel_sequences,502- "keyframe superseding media v2 objects"503- );504- send_tasks.abort_all();505- }506-507- let media_frame = media_frame_for_encoded(508- config,509- &frame,510- replay_discontinuity || scheduler_discontinuity,511- )?;512- let sequence = frame.sequence;513- let keyframe = independently_decodable;514- // Reserve stream IDs in encoded-frame order. Opening them515- // inside the concurrent writer tasks would let task516- // scheduling reorder the QUIC object sequence even though517- // each object carries a monotonic media sequence number.518- let stream = match tokio::time::timeout(519- MEDIA_WRITE_TIMEOUT,520- connection.open_uni(),521- )522- .await523- {524- Ok(Ok(stream)) => stream,525- Ok(Err(error)) => {526- warn!(sequence, %error, "opening media v2 object stream failed");527- if !scheduler.fail(sequence).is_empty() {528- send_tasks.abort_all();529- }530- continue;531- }532- Err(_) => {533- warn!(sequence, "opening media v2 object stream timed out");534- if !scheduler.fail(sequence).is_empty() {535- send_tasks.abort_all();536- }537- continue;538- }539- };540- let stream = ResetOnDropSendStream::new(stream);541- if let Err(error) = stream542- .stream()543- .set_priority(media_v2_priority(keyframe))544- {545- warn!(sequence, %error, "setting media v2 object priority failed");546- if !scheduler.fail(sequence).is_empty() {547- send_tasks.abort_all();548- }549- continue;550- }551- send_tasks.spawn(async move {552- (553- sequence,554- send_media_v2_object(stream, media_frame).await,555- )556- });557- replay_cursor.commit_sent(&frame);558- }559- }560- }561- };562-563- send_tasks.abort_all();564- while send_tasks.join_next().await.is_some() {}565- result566-}567-568-pub(super) fn media_frame_for_encoded(569- _config: &HostConfig,570- frame: &EncodedFrame,571- discontinuity: bool,572-) -> Result<MediaFrame> {573- let mut flags = FrameFlags::NONE;574- if frame.keyframe {575- flags = flags.union(FrameFlags::KEYFRAME);576- }577- if frame.codec_config {578- flags = flags.union(FrameFlags::CODEC_CONFIG);579- }580- if discontinuity || frame.discontinuity {581- flags = flags.union(FrameFlags::DISCONTINUITY);582- }583- let header = MediaFrameHeader::h264(584- frame.width,585- frame.height,586- frame.data.len(),587- frame.sequence,588- frame.capture_timestamp_micros,589- frame.presentation_timestamp_micros,590- flags,591- )?;592- MediaFrame::new(header, frame.data.as_ref().to_vec()).map_err(Into::into)593-}594-595-async fn send_media_v2_object(mut stream: ResetOnDropSendStream, frame: MediaFrame) -> Result<()> {596- tokio::time::timeout(597- MEDIA_WRITE_TIMEOUT,598- write_media_frame(stream.stream_mut(), &frame),599- )600- .await601- .context("timed out writing media v2 object")??;602- stream603- .stream_mut()604- .finish()605- .context("finishing media v2 object stream")?;606- match tokio::time::timeout(MEDIA_V2_PEER_ACK_TIMEOUT, stream.stream().stopped())607- .await608- .context("timed out waiting for media v2 object acknowledgement")?609- .context("waiting for media v2 object acknowledgement")?610- {611- None => {612- stream.disarm();613- Ok(())614- }615- Some(code) => bail!("peer stopped media v2 object stream with code {code}"),616- }617-}618-619-fn media_v2_priority(keyframe: bool) -> i32 {620- if keyframe {621- MEDIA_V2_KEYFRAME_PRIORITY622- } else {623- MEDIA_V2_DELTA_PRIORITY624- }625-}626-627-#[cfg(test)]628-mod tests {629- use super::super::{DropNotify, endpoint};630- use super::*;631-632- #[test]633- fn media_v2_scheduler_is_bounded_and_drops_a_saturated_delta_suffix() {634- let mut scheduler = MediaV2Scheduler::default();635- assert_eq!(636- scheduler.schedule(10, true),637- MediaV2ScheduleDecision::Send {638- discontinuity: false,639- cancel_sequences: vec![],640- }641- );642- for sequence in 11..14 {643- assert!(matches!(644- scheduler.schedule(sequence, false),645- MediaV2ScheduleDecision::Send { .. }646- ));647- }648- assert_eq!(scheduler.in_flight.len(), MEDIA_V2_IN_FLIGHT_CAPACITY);649-650- assert_eq!(651- scheduler.schedule(14, false),652- MediaV2ScheduleDecision::EnterResync {653- cancel_sequences: vec![10, 11, 12, 13],654- }655- );656- assert!(scheduler.in_flight.is_empty());657- assert_eq!(658- scheduler.schedule(15, false),659- MediaV2ScheduleDecision::SkipUntilKeyframe660- );661- }662-663- #[test]664- fn blocked_old_streams_cannot_prevent_a_keyframe_decision() {665- let mut scheduler = MediaV2Scheduler::default();666- for sequence in 20..24 {667- assert!(matches!(668- scheduler.schedule(sequence, sequence == 20),669- MediaV2ScheduleDecision::Send { .. }670- ));671- }672- assert_eq!(scheduler.in_flight.len(), MEDIA_V2_IN_FLIGHT_CAPACITY);673-674- assert_eq!(675- scheduler.schedule(30, true),676- MediaV2ScheduleDecision::Send {677- discontinuity: true,678- cancel_sequences: vec![20, 21, 22, 23],679- }680- );681- assert_eq!(scheduler.in_flight, vec![30]);682- }683-684- #[test]685- fn media_v2_send_failure_cancels_dependents_and_resyncs_on_keyframe() {686- let mut scheduler = MediaV2Scheduler::default();687- assert!(matches!(688- scheduler.schedule(40, true),689- MediaV2ScheduleDecision::Send { .. }690- ));691- assert!(matches!(692- scheduler.schedule(41, false),693- MediaV2ScheduleDecision::Send { .. }694- ));695-696- assert_eq!(scheduler.fail(40), vec![41]);697- assert_eq!(698- scheduler.schedule(42, false),699- MediaV2ScheduleDecision::SkipUntilKeyframe700- );701- assert_eq!(702- scheduler.schedule(50, true),703- MediaV2ScheduleDecision::Send {704- discontinuity: true,705- cancel_sequences: vec![],706- }707- );708- scheduler.complete(50);709- assert!(scheduler.in_flight.is_empty());710- assert!(matches!(711- scheduler.schedule(51, false),712- MediaV2ScheduleDecision::Send {713- discontinuity: false,714- ..715- }716- ));717- }718-719- #[test]720- fn cancelled_old_completion_cannot_poison_the_replacement_gop() {721- let mut scheduler = MediaV2Scheduler::default();722- assert!(matches!(723- scheduler.schedule(60, true),724- MediaV2ScheduleDecision::Send { .. }725- ));726- assert!(matches!(727- scheduler.schedule(61, false),728- MediaV2ScheduleDecision::Send { .. }729- ));730- assert!(matches!(731- scheduler.schedule(70, true),732- MediaV2ScheduleDecision::Send {733- discontinuity: true,734- ..735- }736- ));737-738- assert!(scheduler.fail(60).is_empty());739- assert!(matches!(740- scheduler.schedule(71, false),741- MediaV2ScheduleDecision::Send {742- discontinuity: false,743- ..744- }745- ));746- }747-748- #[test]749- fn media_v2_keyframes_have_strictly_higher_transport_priority() {750- assert!(media_v2_priority(true) > media_v2_priority(false));751- }752-753- #[tokio::test]754- async fn no_frame_peer_drop_reaps_source_and_allows_reconnect() {755- let sessions = Arc::new(SessionRegistry::default());756- let remote = endpoint(1);757- let nonce = [7; 16];758- let media = sessions759- .claim(remote, nonce, InvitationGrants::ALL)760- .unwrap();761- let input = sessions.claim_input(remote, nonce).unwrap();762- let session_id = media.session_id;763-764- let input_sessions = Arc::clone(&sessions);765- let input_shutdown = tokio::spawn(async move {766- loop {767- let notified = input_sessions.session_changed.notified();768- if !input_sessions.is_active(remote, session_id) {769- break;770- }771- notified.await;772- }773- drop(input);774- });775-776- let (_frame_sender, mut frame_receiver) =777- tokio::sync::watch::channel(Option::<EncodedFrame>::None);778- let (source_started_tx, source_started_rx) = tokio::sync::oneshot::channel();779- let (source_reaped_tx, source_reaped_rx) = tokio::sync::oneshot::channel();780- let source_task = tokio::spawn(async move {781- let _notify = DropNotify(Some(source_reaped_tx));782- let _ = source_started_tx.send(());783- std::future::pending::<Result<()>>().await784- });785- let source_task = SourceTaskGuard::new(source_task);786- source_started_rx.await.unwrap();787-788- let (peer_alive, peer_disconnected) = tokio::sync::oneshot::channel::<()>();789- let disconnected = async move {790- let _ = peer_disconnected.await;791- };792- tokio::pin!(disconnected);793- drop(peer_alive);794-795- let activity = tokio::time::timeout(796- Duration::from_millis(100),797- wait_for_media_activity(&mut frame_receiver, disconnected.as_mut()),798- )799- .await800- .expect("no-frame media loop ignored peer disconnect")801- .unwrap();802- assert_eq!(activity, MediaActivity::PeerDisconnected);803-804- source_task.abort_and_wait().await;805- tokio::time::timeout(Duration::from_millis(100), source_reaped_rx)806- .await807- .expect("source task was not reaped after peer disconnect")808- .unwrap();809- drop(media);810- tokio::time::timeout(Duration::from_millis(100), input_shutdown)811- .await812- .expect("input lease did not observe media shutdown")813- .unwrap();814-815- assert!(816- sessions817- .claim(endpoint(2), [8; 16], InvitationGrants::ALL)818- .is_ok()819- );820- }821-}
M crates/sigil-host/src/server/mod.rs +2 -7
@@ -1,5 +1,4 @@
11 use std::future::Future;2-use std::pin::Pin;32 use std::sync::Arc;43 use std::sync::atomic::Ordering;54 use std::time::{Duration, Instant};
@@ -21,7 +20,7 @@ use sigil_protocol::{
2120 MediaFrame, MediaFrameHeader, MediaObjectHeaderV3, MediaObjectV3, encode_media_frame_object,2221 media_moq_broadcast_name, read_client_hello, read_input_event, read_media_control_request_v3,2322 read_media_feedback_report_v1, write_adaptive_bitrate_decision_v1, write_host_hello,24- write_input_ack, write_media_frame, write_media_object_v3,23+ write_input_ack, write_media_object_v3,2524 };2625 use tracing::{debug, error, info, warn};2726
@@ -45,24 +44,20 @@ const ENCODER_CONTROL_COMMIT_TIMEOUT: Duration = Duration::from_secs(2);
4544 4645 mod adaptive;4746 mod handlers;48-mod media_v2;4947 mod media_v3;5048 mod moq;5149 mod session;5250 mod startup;5351 54-#[allow(unused_imports)]55-pub(crate) use adaptive::MotionResolutionPolicy;5652 pub(crate) use adaptive::VideoDimensions;5753 use adaptive::serve_media_feedback;5854 pub use handlers::{5955 AudioHandler, AuthorizedMoqHandler, ControlHandler, InputHandler, MediaFeedbackHandler,60- MediaHandler, MediaV2Handler, MediaV3Handler,56+ MediaV3Handler,6157 };6258 use handlers::{6359 HANDSHAKE_TIMEOUT, MEDIA_CAPABILITIES, negotiated_capabilities, receive_hello, send_rejection,6460 };65-use media_v2::{media_frame_for_encoded, serve_media, serve_media_v2};6661 use media_v3::{6762 MediaV3GroupCursor, forward_media_v3_control_requests, new_current_gop_frames, serve_media_v3,6863 };
M crates/sigil-host/src/server/moq.rs +27 -0
M crates/sigil-protocol/src/lib.rs +1 -18
@@ -41,8 +41,7 @@ pub use invitation::{
4141 };4242 pub use media::{4343 FrameFlags, MEDIA_HEADER_LEN, MediaCodec, MediaFrame, MediaFrameHeader,44- decode_media_frame_object, encode_media_frame_object, read_media_frame, read_media_object,45- write_media_frame,44+ decode_media_frame_object, encode_media_frame_object,4645 };4746 pub use media_v3::{4847 KeyframeRequestReasonV3, MAX_MEDIA_GROUP_BYTES_V3, MAX_MEDIA_OBJECT_DELIVERY_TIMEOUT_MS,
@@ -60,10 +59,6 @@ pub use moq_catalog::{
6059 /// Protocol version encoded in v1 messages.6160 pub const PROTOCOL_VERSION: u16 = 1;6261 63-/// ALPN for the v1 encoded video stream.64-pub const MEDIA_ALPN_V1: &[u8] = b"sigil/media/1";65-/// ALPN for one encoded video object per host-opened unidirectional stream.66-pub const MEDIA_ALPN_V2: &[u8] = b"sigil/media/2";6762 /// ALPN for custom MoQ-style grouped media objects and keyframe control.6863 ///6964 /// This is a Sigil protocol and is not IETF MoQ Transport compatible.
@@ -79,14 +74,6 @@ pub const MOQ_VIDEO_H264_TRACK: &str = "video/h264";
7974 /// ALPN for the v1 low-latency Opus datagram connection.8075 pub const AUDIO_ALPN_V1: &[u8] = b"sigil/audio/1";8176 82-/// The inherited frame-stream ALPN, provided only for an explicit migration83-/// adapter. New peers should prefer [`MEDIA_ALPN_V2`] and may retain84-/// [`MEDIA_ALPN_V1`] only as a compatibility fallback.85-pub const LEGACY_FRAME_ALPN_V0: &[u8] = b"sigil/frame-stream/0";86-/// The inherited input-stream ALPN, provided only for an explicit migration87-/// adapter. New peers must advertise [`INPUT_ALPN_V1`].88-pub const LEGACY_INPUT_ALPN_V0: &[u8] = b"sigil/input-stream/0";89-9077 /// Maximum encoded video access-unit size accepted from the network (16 MiB).9178 pub const MAX_MEDIA_PAYLOAD_LEN: usize = 16 * 1024 * 1024;9279 /// Maximum dimension carried by a v1 video header.
@@ -124,15 +111,11 @@ mod tests {
124111 125112 #[test]126113 fn alpn_values_are_deployment_golden_vectors() {127- assert_eq!(MEDIA_ALPN_V1, b"sigil/media/1");128- assert_eq!(MEDIA_ALPN_V2, b"sigil/media/2");129114 assert_eq!(MEDIA_ALPN_V3, b"sigil/media/3");130115 assert_eq!(MEDIA_FEEDBACK_ALPN_V1, b"sigil/media-feedback/1");131116 assert_eq!(INPUT_ALPN_V1, b"sigil/input/1");132117 assert_eq!(CONTROL_ALPN_V1, b"sigil/control/1");133118 assert_eq!(AUDIO_ALPN_V1, b"sigil/audio/1");134- assert_eq!(LEGACY_FRAME_ALPN_V0, b"sigil/frame-stream/0");135- assert_eq!(LEGACY_INPUT_ALPN_V0, b"sigil/input-stream/0");136119 }137120 138121 #[test]
M crates/sigil-protocol/src/media.rs +0 -144
@@ -1,5 +1,3 @@
1-use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};2-31 use crate::{42 MAX_MEDIA_PAYLOAD_LEN, MAX_VIDEO_DIMENSION, MAX_VIDEO_PIXELS, PROTOCOL_VERSION, ProtocolError,53 Result,
@@ -274,74 +272,8 @@ impl MediaFrame {
274272 }275273 }276274 277-/// Read one complete frame. A clean EOF before the next header returns `None`.278-pub async fn read_media_frame<R>(reader: &mut R) -> Result<Option<MediaFrame>>279-where280- R: AsyncRead + Unpin,281-{282- let mut wire = [0; MEDIA_HEADER_LEN];283- let read = reader.read(&mut wire[..1]).await?;284- if read == 0 {285- return Ok(None);286- }287- reader.read_exact(&mut wire[1..]).await?;288- let header = MediaFrameHeader::decode(&wire)?;289-290- // Header validation happens before allocation.291- let mut payload = vec![0; header.payload_len as usize];292- reader.read_exact(&mut payload).await?;293- Ok(Some(MediaFrame { header, payload }))294-}295-296-/// Read exactly one complete frame followed by a clean stream EOF.297-///298-/// This is the object boundary used by the v2 media ALPN. An empty stream or299-/// any bytes after the declared frame are rejected so one QUIC stream can300-/// never be interpreted as zero or multiple media objects.301-pub async fn read_media_object<R>(reader: &mut R) -> Result<MediaFrame>302-where303- R: AsyncRead + Unpin,304-{305- let frame = read_media_frame(reader)306- .await?307- .ok_or(ProtocolError::InvalidMessage {308- message_type: "media object",309- reason: "stream ended before the media frame",310- })?;311- let mut trailing = [0_u8; 1];312- if reader.read(&mut trailing).await? != 0 {313- return Err(ProtocolError::InvalidMessage {314- message_type: "media object",315- reason: "trailing bytes after the media frame",316- });317- }318- Ok(frame)319-}320-321-/// Validate and write one complete frame.322-pub async fn write_media_frame<W>(writer: &mut W, frame: &MediaFrame) -> Result<()>323-where324- W: AsyncWrite + Unpin,325-{326- frame.header.validate()?;327- if frame.payload.len() != frame.header.payload_len as usize {328- return Err(ProtocolError::InvalidMessage {329- message_type: "media frame",330- reason: "payload does not match declared length",331- });332- }333- writer.write_all(&frame.header.encode()?).await?;334- writer.write_all(&frame.payload).await?;335- writer.flush().await?;336- Ok(())337-}338-339275 #[cfg(test)]340276 mod tests {341- use std::io::ErrorKind;342-343- use tokio::io::{AsyncWriteExt, duplex};344-345277 use super::*;346278 347279 fn header() -> MediaFrameHeader {
@@ -415,30 +347,6 @@ mod tests {
415347 );416348 }417349 418- #[tokio::test]419- async fn async_frame_round_trip_and_clean_eof() {420- let expected = MediaFrame::new(header(), vec![0, 0, 0, 1]).unwrap();421- let (mut sender, mut receiver) = duplex(256);422- write_media_frame(&mut sender, &expected).await.unwrap();423- sender.shutdown().await.unwrap();424-425- assert_eq!(426- read_media_frame(&mut receiver).await.unwrap(),427- Some(expected)428- );429- assert_eq!(read_media_frame(&mut receiver).await.unwrap(), None);430- }431-432- #[tokio::test]433- async fn media_object_requires_exactly_one_frame_and_clean_eof() {434- let expected = MediaFrame::new(header(), vec![0, 0, 0, 1]).unwrap();435- let (mut sender, mut receiver) = duplex(256);436- write_media_frame(&mut sender, &expected).await.unwrap();437- sender.shutdown().await.unwrap();438-439- assert_eq!(read_media_object(&mut receiver).await.unwrap(), expected);440- }441-442350 #[test]443351 fn object_transport_envelope_round_trips_exactly() {444352 let frame = MediaFrame::new(header(), vec![0, 0, 0, 1]).unwrap();
@@ -451,56 +359,4 @@ mod tests {
451359 assert!(decode_media_frame_object(&trailing).is_err());452360 assert!(decode_media_frame_object(&object[..MEDIA_HEADER_LEN - 1]).is_err());453361 }454-455- #[tokio::test]456- async fn media_object_rejects_empty_and_trailing_streams() {457- let (mut empty_sender, mut empty_receiver) = duplex(16);458- empty_sender.shutdown().await.unwrap();459- assert!(matches!(460- read_media_object(&mut empty_receiver).await,461- Err(ProtocolError::InvalidMessage {462- message_type: "media object",463- ..464- })465- ));466-467- let frame = MediaFrame::new(header(), vec![0, 0, 0, 1]).unwrap();468- let (mut sender, mut receiver) = duplex(256);469- write_media_frame(&mut sender, &frame).await.unwrap();470- sender.write_all(&[0xff]).await.unwrap();471- sender.shutdown().await.unwrap();472- assert!(matches!(473- read_media_object(&mut receiver).await,474- Err(ProtocolError::InvalidMessage {475- message_type: "media object",476- ..477- })478- ));479- }480-481- #[tokio::test]482- async fn truncated_payload_is_an_io_error() {483- let (mut sender, mut receiver) = duplex(256);484- sender.write_all(&header().encode().unwrap()).await.unwrap();485- sender.write_all(&[1, 2]).await.unwrap();486- sender.shutdown().await.unwrap();487-488- let error = read_media_frame(&mut receiver).await.unwrap_err();489- assert!(490- matches!(error, ProtocolError::Io(ref io) if io.kind() == ErrorKind::UnexpectedEof)491- );492- }493-494- #[tokio::test]495- async fn oversized_payload_is_rejected_before_body_read() {496- let mut wire = header().encode().unwrap();497- wire[16..20].copy_from_slice(&((MAX_MEDIA_PAYLOAD_LEN as u32) + 1).to_be_bytes());498- let (mut sender, mut receiver) = duplex(128);499- sender.write_all(&wire).await.unwrap();500-501- assert!(matches!(502- read_media_frame(&mut receiver).await,503- Err(ProtocolError::InvalidMediaPayloadLength { .. })504- ));505- }506362 }
M portal/connection-state.mjs +1 -3
M portal/connection-state.test.mjs +0 -3
M portal/enrollment-ui.test.mjs +0 -7
M portal/frame-session.mjs +1 -37
@@ -8,7 +8,6 @@ export function newFrameSession() {
88 return {99 expectedGeneration: null,1010 pendingAcknowledgments: [],11- pendingLegacyFrames: [],1211 pendingFrameErrors: [],1312 failed: false,1413 failureDetail: null,
@@ -47,44 +46,9 @@ export function activateFrameSession(session, generation) {
4746 (pendingGeneration) => pendingGeneration ?? generation,4847 );4948 session.pendingAcknowledgments.length = 0;50- const legacyFrames = session.pendingLegacyFrames.splice(0);51- acknowledgments.push(...legacyFrames.map((frame) => frame.generation));52- return { acknowledgments, legacyFrames };49+ return { acknowledgments };5350 }5451 5552 export function isActiveFrameSession(session, activeSession) {5653 return session !== null && session === activeSession && !session.closing;5754 }58-59-export function stageLegacyFrame(session, payload) {60- if (!session || session.closing || !validGeneration(payload?.generation)) {61- throw new TypeError('invalid legacy frame delivery');62- }63- const generation = payload.generation;64- if (session.expectedGeneration !== null) {65- return {66- acknowledgments: [generation],67- staged: false,68- accepted: generation === session.expectedGeneration,69- };70- }71-72- const highestPendingGeneration = session.pendingLegacyFrames.reduce(73- (highest, frame) => Math.max(highest, frame.generation),74- 0,75- );76- if (generation < highestPendingGeneration) {77- return { acknowledgments: [generation], staged: false, accepted: false };78- }79-80- const acknowledgments = [];81- if (generation > highestPendingGeneration && highestPendingGeneration > 0) {82- acknowledgments.push(...session.pendingLegacyFrames.map((frame) => frame.generation));83- session.pendingLegacyFrames.length = 0;84- }85- if (session.pendingLegacyFrames.length >= FRAME_CHANNEL_CAPACITY) {86- throw new Error('pre-connect legacy frame capacity exceeded');87- }88- session.pendingLegacyFrames.push(payload);89- return { acknowledgments, staged: true, accepted: false };90-}
M portal/frame-session.test.mjs +0 -27
@@ -7,7 +7,6 @@ import {
77 isActiveFrameSession,88 newFrameSession,99 stageFrameAcknowledgment,10- stageLegacyFrame,1110 } from './frame-session.mjs';1211 1312 test('pre-result acknowledgments stay bounded and flush with the active generation', () => {
@@ -18,36 +17,10 @@ test('pre-result acknowledgments stay bounded and flush with the active generati
1817 assert.throws(() => stageFrameAcknowledgment(session), /capacity exceeded/);1918 assert.deepEqual(activateFrameSession(session, 12), {2019 acknowledgments: [12, 12, 12, 12],21- legacyFrames: [],2220 });2321 assert.equal(stageFrameAcknowledgment(session), 12);2422 });2523 26-test('higher-generation legacy frames evict stale pre-result capacity', () => {27- const session = newFrameSession();28- for (let index = 0; index < FRAME_CHANNEL_CAPACITY; index++) {29- assert.deepEqual(stageLegacyFrame(session, { generation: 8, sequence: index }), {30- acknowledgments: [],31- staged: true,32- accepted: false,33- });34- }35- assert.deepEqual(stageLegacyFrame(session, { generation: 9, sequence: 1 }), {36- acknowledgments: [8, 8, 8, 8],37- staged: true,38- accepted: false,39- });40- assert.deepEqual(stageLegacyFrame(session, { generation: 8, sequence: 99 }), {41- acknowledgments: [8],42- staged: false,43- accepted: false,44- });45- assert.deepEqual(activateFrameSession(session, 9), {46- acknowledgments: [9],47- legacyFrames: [{ generation: 9, sequence: 1 }],48- });49-});50-5124 test('only the current non-closing frame session owns callbacks', () => {5225 const current = newFrameSession();5326 const stale = newFrameSession();
M portal/index.html +0 -1
M portal/main.js +4 -17
@@ -261,12 +261,6 @@ async function cancelInvitation() {
261261 async function checkDevelopmentConnectionMode() {262262 const mode = await invoke('development_connection_mode');263263 connectionState.setDevelopmentMode(mode.enabled);264- if (mode.force_jpeg) {265- const jpegBadge = document.getElementById('dev-jpeg-badge');266- jpegBadge.classList.remove('hidden');267- jpegBadge.title = mode.jpeg_warning;268- console.warn('[development JPEG compatibility mode]', mode.jpeg_warning);269- }270264 if (!mode.enabled) return;271265 272266 const badge = document.getElementById('dev-connect-badge');
@@ -578,10 +572,10 @@ function togglePanel(force) {
578572 }579573 580574 // ─── WebCodecs detection ──────────────────────────────────────────────────────581-// Finish the codec probe and publish the same delivery mode to Rust before582-// exposing any connect handlers. This keeps raw WebCodecs frames and the Rust583-// JPEG compatibility path from disagreeing during startup.584-const hasWebCodecs = await detectAndPublishVideoDeliveryMode({ invokeCommand: invoke });575+// Finish the codec probe and publish the result to Rust before exposing any576+// connect handlers. Rust refuses to connect when WebCodecs is unavailable, so577+// the pipeline below can assume binary WebCodecs delivery.578+await detectAndPublishVideoDeliveryMode({ invokeCommand: invoke });585579 586580 // ─── Frame decoding ───────────────────────────────────────────────────────────587581 const canvas = document.getElementById('frame-canvas');
@@ -628,7 +622,6 @@ controlRuntime = createControlRuntime({
628622 onReleaseFailure: () => setStatus('err', 'cursor release failed · quit app'),629623 });630624 videoPipeline = createVideoPipelineSession({631- hasWebCodecs,632625 canvas,633626 context: ctx,634627 requestKeyframe: (reason) => streamRuntime?.requestKeyframe(reason),
@@ -766,12 +759,6 @@ setInterval(() => {
766759 if (connectionState.connected) updateStreamStats();767760 }, 250);768761 769-// The software decoder/JPEG compatibility path intentionally remains an event:770-// it is only selected when WebCodecs is unavailable and is not latency-critical.771-listen('frame', (event) => {772- streamRuntime.handleLegacyFrame(event.payload);773-});774-775762 listen('frame-stats', (event) => {776763 if (streamRuntime.handleFrameStats(event.payload)) updateStreamStats();777764 });
M portal/stream-runtime.mjs +0 -24
@@ -9,7 +9,6 @@ import {
99 isActiveFrameSession,1010 newFrameSession,1111 stageFrameAcknowledgment,12- stageLegacyFrame,1312 } from './frame-session.mjs';1413 import {1514 AdaptiveFeedbackPublisher,
@@ -131,11 +130,6 @@ export function createStreamRuntime({
131130 (payload) => isCurrentFrameGeneration(payload?.generation, generation),132131 );133132 if (pendingError) throw new Error(pendingError.error || 'Media connection failed');134- for (const payload of activation.legacyFrames) {135- if (isCurrentFrameGeneration(payload?.generation, generation)) {136- videoPipeline.processFramePayload(payload);137- }138- }139133 adaptiveFeedbackAvailable = feedbackAvailable === true;140134 adaptiveFeedbackError = typeof feedbackError === 'string' ? feedbackError : null;141135 feedbackPublisher.start(generation, adaptiveFeedbackAvailable);
@@ -190,7 +184,6 @@ export function createStreamRuntime({
190184 videoPipeline.processFramePayload({191185 width: frame.width,192186 height: frame.height,193- data: null,194187 codec: frame.codec,195188 keyframe: frame.keyframe,196189 codecConfig: frame.codecConfig,
@@ -215,22 +208,6 @@ export function createStreamRuntime({
215208 }216209 }217210 218- function handleLegacyFrame(payload) {219- const session = activeFrameSession;220- if (!isActiveFrameSession(session, activeFrameSession)) return false;221- try {222- const delivery = stageLegacyFrame(session, payload);223- for (const generation of delivery.acknowledgments) {224- sendFrameAcknowledgment(generation);225- }226- if (delivery.accepted) videoPipeline.processFramePayload(payload);227- return delivery.accepted;228- } catch (error) {229- failFrameSession(session, error);230- return false;231- }232- }233-234211 function handleFrameStats(payload) {235212 if (!isCurrentFrameGeneration(payload?.generation, activeFrameGeneration)) return false;236213 const diagnostics = normalizeFrameStatsPayload(payload);
@@ -348,7 +325,6 @@ export function createStreamRuntime({
348325 teardown,349326 requestKeyframe,350327 handleBinaryFrame,351- handleLegacyFrame,352328 handleFrameStats,353329 handleAdaptiveDecision,354330 handleAdaptiveFeedbackState,
M portal/stream-runtime.test.mjs +5 -22
@@ -128,30 +128,26 @@ test('closing a partially constructed attempt falls back to the active frame ses
128128 assert.equal(runtime.generation, null);129129 });130130 131-test('activation flushes binary and legacy acknowledgments before staged legacy decode', () => {131+test('activation flushes pre-connect binary acknowledgments', () => {132132 const { runtime, events } = subject();133133 runtime.prepareConnection();134134 const session = runtime.openFrameSession();135135 assert.equal(runtime.handleBinaryFrame(frameEnvelope(), session), true);136- assert.equal(runtime.handleLegacyFrame({ generation: 7, sequence: 12, data: 'jpeg' }), false);137136 138137 events.length = 0;139138 activate(runtime, session, 7);140139 assert.deepEqual(events.map((event) => event.type), [141140 'invoke',142- 'invoke',143- 'process',144141 'publisher-start',145142 ]);146- assert.deepEqual(events.slice(0, 2).map((event) => event.args.generation), [7, 7]);147- assert.equal(events[2].payload.sequence, 12);143+ assert.equal(events[0].args.generation, 7);148144 });149145 150-test('a current pending frame error aborts activation after acknowledgments and before legacy decode', () => {146+test('a current pending frame error aborts activation after acknowledgments', () => {151147 const { runtime, events } = subject();152148 runtime.prepareConnection();153149 const session = runtime.openFrameSession();154- runtime.handleLegacyFrame({ generation: 9, sequence: 1, data: 'jpeg' });150+ runtime.handleBinaryFrame(frameEnvelope(), session);155151 runtime.handleFrameError({ generation: 9, error: 'native failed' });156152 events.length = 0;157153
@@ -181,19 +177,6 @@ test('active binary delivery acknowledges before decode and malformed delivery s
181177 assert.equal(events[2].args.generation, 5);182178 });183179 184-test('legacy deliveries acknowledge every active generation but decode only the current one', () => {185- const { runtime, events } = subject();186- runtime.prepareConnection();187- const session = runtime.openFrameSession();188- activate(runtime, session, 8);189- events.length = 0;190-191- assert.equal(runtime.handleLegacyFrame({ generation: 7, sequence: 1 }), false);192- assert.equal(runtime.handleLegacyFrame({ generation: 8, sequence: 2 }), true);193- assert.deepEqual(events.map((event) => event.type), ['invoke', 'invoke', 'process']);194- assert.deepEqual(events.slice(0, 2).map((event) => event.args.generation), [7, 8]);195-});196-197180 test('pre-connect frame errors are bounded and stale active errors are ignored', () => {198181 const { runtime, events } = subject({ connected: true });199182 runtime.prepareConnection();
@@ -336,7 +319,7 @@ test('adaptive feedback preserves bounded queue fallbacks and pressure signals',
336319 });337320 });338321 339-test('adaptive feedback projects the bounded JPEG decode-plus-latest queue and overwrite counter', () => {322+test('adaptive feedback projects the bounded presenter queue and overwrite counter', () => {340323 const { runtime, events } = subject();341324 runtime.prepareConnection();342325 const session = runtime.openFrameSession();
M portal/video-capability.mjs +1 -4
M portal/video-capability.test.mjs +4 -7
@@ -77,9 +77,8 @@ test('awaits the probe before publishing the exact raw delivery mode', async ()
7777 ]);7878 });7979 80-test('keeps the real probe but obeys native forced-JPEG publication', async () => {80+test('keeps the real probe but obeys native publication of the effective mode', async () => {8181 const calls = [];82- const warnings = [];8382 class SupportedDecoder {8483 static async isConfigSupported(config) {8584 calls.push(['probe', config]);
@@ -93,7 +92,7 @@ test('keeps the real probe but obeys native forced-JPEG publication', async () =
9392 calls.push(args);9493 return false;9594 },96- logger: { warn: (...args) => warnings.push(args), error() {} },95+ logger: { warn() {}, error() {} },9796 }), false);9897 assert.deepEqual(calls, [9998 ['probe', {
@@ -102,11 +101,9 @@ test('keeps the real probe but obeys native forced-JPEG publication', async () =
102101 }],103102 ['set_webcodecs_available', { available: true }],104103 ]);105- assert.equal(warnings.length, 1);106- assert.match(warnings[0][0], /JPEG compatibility mode forced/);107104 });108105 109-test('keeps frontend on JPEG when capability publication fails', async () => {106+test('reports unavailable when capability publication fails', async () => {110107 const errors = [];111108 class SupportedDecoder {112109 static async isConfigSupported() { return { supported: true }; }
@@ -118,7 +115,7 @@ test('keeps frontend on JPEG when capability publication fails', async () => {
118115 logger: { warn() {}, error: (...args) => errors.push(args) },119116 }), false);120117 assert.equal(errors.length, 1);121- assert.match(errors[0][0], /using JPEG fallback/);118+ assert.match(errors[0][0], /could not publish WebCodecs capability/);122119 });123120 124121 test('rejects a missing native publication boundary', async () => {
M portal/video-pipeline.mjs +77 -220
@@ -23,10 +23,8 @@ import {
2323 export const MAX_DECODE_QUEUE_SIZE = 2;2424 const MAX_DECODE_TIMINGS = 8;2525 const PRESENTER_QUEUE_CAPACITY = 2;26-const JPEG_PRESENTER_QUEUE_CAPACITY = 2;2726 2827 export function createVideoPipelineSession({29- hasWebCodecs,3028 canvas,3129 context,3230 requestKeyframe = () => {},
@@ -36,12 +34,6 @@ export function createVideoPipelineSession({
3634 now = () => performance.now(),3735 createDecoder = (callbacks) => new VideoDecoder(callbacks),3836 createEncodedChunk = (init) => new EncodedVideoChunk(init),39- decodeBase64 = (data) => Uint8Array.from(atob(data), c => c.charCodeAt(0)),40- createBlob = (parts, options) => new Blob(parts, options),41- createObjectUrl = (blob) => URL.createObjectURL(blob),42- revokeObjectUrl = (url) => URL.revokeObjectURL(url),43- createImage = () => new Image(),44- abortImage = (image) => image.removeAttribute('src'),4537 requestFrame = (callback) => requestAnimationFrame(callback),4638 cancelFrame = (handle) => cancelAnimationFrame(handle),4739 setTimer = (callback, delayMs) => setTimeout(callback, delayMs),
@@ -60,9 +52,6 @@ export function createVideoPipelineSession({
6052 let presentedFrames = 0;6153 let presentationDroppedFrames = 0;6254 let lastVideoDrawCompletedAtMs = null;63- let jpegFrameSequence = 0;64- let activeJpegDecode = null;65- let queuedJpegFrame = null;6655 6756 const frontendDeliveryRate = new RollingRateWindow();6857 const decoderInputRate = new RollingRateWindow();
@@ -118,107 +107,8 @@ export function createVideoPipelineSession({
118107 resetAudioSync(...args);119108 }120109 121- function releaseActiveJpegDecode(frame, { abort = false } = {}) {122- if (!frame || frame.released) return;123- frame.released = true;124- frame.image.onload = null;125- frame.image.onerror = null;126- if (abort) {127- try {128- abortImage(frame.image);129- } catch (error) {130- console.warn('could not abort stale JPEG decode:', error);131- }132- }133- revokeObjectUrl(frame.url);134- }135-136- function startQueuedJpegDecode() {137- if (activeJpegDecode || !queuedJpegFrame) return;138- const pending = queuedJpegFrame;139- queuedJpegFrame = null;140-141- let url = null;142- let image;143- try {144- url = createObjectUrl(pending.blob);145- image = createImage();146- } catch (error) {147- if (url !== null) revokeObjectUrl(url);148- droppedFrames++;149- console.warn('could not start JPEG decode:', error);150- startQueuedJpegDecode();151- return;152- }153-154- const frame = {155- ...pending,156- image,157- url,158- released: false,159- superseded: false,160- };161- activeJpegDecode = frame;162- image.onload = () => {163- if (activeJpegDecode !== frame) return;164- activeJpegDecode = null;165- try {166- if (!frame.superseded && queuedJpegFrame === null) {167- const drawStartedAt = now();168- context.drawImage(image, 0, 0, canvas.width, canvas.height);169- const presentedAt = now();170- presentedFrames++;171- presentationRate.record(presentedAt);172- presentationCadence.record(presentedAt);173- drawLatency.record(presentedAt - drawStartedAt, presentedAt);174- clientPresentationLatency.record(175- presentedAt - frame.receivedAt,176- presentedAt,177- );178- }179- } finally {180- releaseActiveJpegDecode(frame);181- startQueuedJpegDecode();182- }183- };184- image.onerror = () => {185- if (activeJpegDecode !== frame) return;186- activeJpegDecode = null;187- if (!frame.superseded && queuedJpegFrame === null) droppedFrames++;188- releaseActiveJpegDecode(frame);189- startQueuedJpegDecode();190- };191- try {192- image.src = url;193- } catch (error) {194- if (activeJpegDecode === frame) activeJpegDecode = null;195- droppedFrames++;196- releaseActiveJpegDecode(frame, { abort: true });197- console.warn('could not assign JPEG decode source:', error);198- startQueuedJpegDecode();199- }200- }201-202- function enqueueJpegFrame(frame) {203- if (!activeJpegDecode) {204- queuedJpegFrame = frame;205- startQueuedJpegDecode();206- return;207- }208- if (!activeJpegDecode.superseded) {209- activeJpegDecode.superseded = true;210- presentationDroppedFrames++;211- }212- if (queuedJpegFrame) presentationDroppedFrames++;213- queuedJpegFrame = frame;214- }215-216110 function teardown() {217111 resetAvSync();218- queuedJpegFrame = null;219- const active = activeJpegDecode;220- activeJpegDecode = null;221- releaseActiveJpegDecode(active, { abort: true });222112 framePresenter.clear();223113 decodeTimings.clear();224114 if (videoDecoder) {
@@ -367,14 +257,13 @@ export function createVideoPipelineSession({
367257 return true;368258 }369259 370- function processFramePayload(payload, binaryData = null) {260+ function processFramePayload(payload, binaryData) {371261 const receivedAt = now();372262 frontendDeliveryRate.record(receivedAt);373263 frontendDeliveryCadence.record(receivedAt);374264 const {375265 width,376266 height,377- data,378267 codec = 'h264',379268 keyframe,380269 codecConfig = keyframe,
@@ -387,117 +276,89 @@ export function createVideoPipelineSession({
387276 }388277 receivedFrames++;389278 390- if (hasWebCodecs) {391- // Binary channels provide an exact view over the IPC ArrayBuffer. The392- // base64 branch only supports an older in-process sender during migration.393- const raw = binaryData ?? decodeBase64(data);394- const decoderBacklogged = videoDecoder?.decodeQueueSize >= MAX_DECODE_QUEUE_SIZE;395- if (decoderBacklogged) {396- enterDecoderRecovery(DECODER_RECOVERY_REASONS.FRONTEND_BACKPRESSURE);397- }398- const decoderBoundary = discontinuity || !decoderConfigured;399- let formatPlan;400- try {401- formatPlan = videoFormatTransition.plan({402- sequence,403- codec,404- width,405- height,406- keyframe,407- codecConfig,408- discontinuity: decoderBoundary,279+ // Binary channels provide an exact view over the IPC ArrayBuffer.280+ const raw = binaryData;281+ const decoderBacklogged = videoDecoder?.decodeQueueSize >= MAX_DECODE_QUEUE_SIZE;282+ if (decoderBacklogged) {283+ enterDecoderRecovery(DECODER_RECOVERY_REASONS.FRONTEND_BACKPRESSURE);284+ }285+ const decoderBoundary = discontinuity || !decoderConfigured;286+ let formatPlan;287+ try {288+ formatPlan = videoFormatTransition.plan({289+ sequence,290+ codec,291+ width,292+ height,293+ keyframe,294+ codecConfig,295+ discontinuity: decoderBoundary,296+ });297+ } catch (error) {298+ console.warn('invalid video format transition:', error);299+ droppedFrames++;300+ enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET);301+ return;302+ }303+ if (formatPlan.action === 'drop-stale') {304+ droppedFrames++;305+ return;306+ }307+ if (formatPlan.action === 'recover') {308+ droppedFrames++;309+ enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET);310+ return;311+ }312+ if (formatPlan.reconfigure) {313+ // A delivered discontinuity keyframe is the recovery boundary we were314+ // waiting for, not evidence that another keyframe is missing.315+ if (discontinuity) {316+ enterDecoderRecovery(DECODER_RECOVERY_REASONS.DISCONTINUITY, {317+ requestKeyframe: !keyframe,318+ });319+ } else {320+ enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET, {321+ requestKeyframe: false,409322 });410- } catch (error) {411- console.warn('invalid video format transition:', error);412- droppedFrames++;413- enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET);414- return;415- }416- if (formatPlan.action === 'drop-stale') {417- droppedFrames++;418- return;419323 }420- if (formatPlan.action === 'recover') {421- droppedFrames++;324+ }325+ if (decoderRecovery.shouldDropFrame({ keyframe })) {326+ if (!decoderRecovery.requestIssued) {422327 enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET);423- return;424- }425- if (formatPlan.reconfigure) {426- // A delivered discontinuity keyframe is the recovery boundary we were427- // waiting for, not evidence that another keyframe is missing.428- if (discontinuity) {429- enterDecoderRecovery(DECODER_RECOVERY_REASONS.DISCONTINUITY, {430- requestKeyframe: !keyframe,431- });432- } else {433- enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET, {434- requestKeyframe: false,435- });436- }437- }438- if (decoderRecovery.shouldDropFrame({ keyframe })) {439- if (!decoderRecovery.requestIssued) {440- enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET);441- }442- droppedFrames++;443- return;444328 }445- const mediaPtsMicros = exactMediaTimestampMicros(ptsMicros);446- const chunkTimestamp = mediaPtsMicros !== null447- ? mediaPtsMicros448- : now() * 1000;329+ droppedFrames++;330+ return;331+ }332+ const mediaPtsMicros = exactMediaTimestampMicros(ptsMicros);333+ const chunkTimestamp = mediaPtsMicros !== null334+ ? mediaPtsMicros335+ : now() * 1000;449336 450- const nals = parseAnnexBNals(raw);451- if (keyframe) {452- let sps = null, pps = null;453- for (const nal of nals) {454- const type = h264NalType(nal);455- if (type === 7) sps = nal;456- else if (type === 8) pps = nal;457- }458- if (sps && pps && formatPlan.reconfigure) {459- initWebCodecsDecoder(width, height, buildAvcDescription(sps, pps), avcCodecStr(sps));460- }461- }462- if (!decoderConfigured || !videoDecoder) { droppedFrames++; return; }463- const sliceNals = nals.filter(nal => {337+ const nals = parseAnnexBNals(raw);338+ if (keyframe) {339+ let sps = null, pps = null;340+ for (const nal of nals) {464341 const type = h264NalType(nal);465- return type !== 7 && type !== 8 && type !== 9;466- });467- if (sliceNals.length > 0) {468- const chunk = createEncodedChunk({469- type: keyframe ? 'key' : 'delta',470- timestamp: chunkTimestamp,471- data: nalsToLengthPrefixed(sliceNals),472- });473- const enqueued = enqueueVideoChunk(chunk, receivedAt, 'h264', mediaPtsMicros);474- commitVideoFrame(formatPlan, keyframe, enqueued);342+ if (type === 7) sps = nal;343+ else if (type === 8) pps = nal;475344 }476- } else {477- // JPEG fallback (openh264 decode → JPEG encode in Rust)478- const dimensionsChanged = frameWidth !== width || frameHeight !== height;479- frameWidth = width;480- frameHeight = height;481- if (canvas.width !== width) canvas.width = width;482- if (canvas.height !== height) canvas.height = height;483- if (dimensionsChanged) {484- onFormatChanged({485- codec: activeCodec,486- width: frameWidth,487- height: frameHeight,488- epoch: activeVideoFormatEpoch,489- });345+ if (sps && pps && formatPlan.reconfigure) {346+ initWebCodecsDecoder(width, height, buildAvcDescription(sps, pps), avcCodecStr(sps));490347 }491- // Keep exactly one browser decode active and one coalesced latest payload.492- // Receipt order is authoritative: once a newer payload is queued, the493- // active decode may finish to release its resources but can never draw.494- const bytes = decodeBase64(data);495- const blob = createBlob([bytes], { type: 'image/jpeg' });496- enqueueJpegFrame({497- sequence: ++jpegFrameSequence,498- blob,499- receivedAt,348+ }349+ if (!decoderConfigured || !videoDecoder) { droppedFrames++; return; }350+ const sliceNals = nals.filter(nal => {351+ const type = h264NalType(nal);352+ return type !== 7 && type !== 8 && type !== 9;353+ });354+ if (sliceNals.length > 0) {355+ const chunk = createEncodedChunk({356+ type: keyframe ? 'key' : 'delta',357+ timestamp: chunkTimestamp,358+ data: nalsToLengthPrefixed(sliceNals),500359 });360+ const enqueued = enqueueVideoChunk(chunk, receivedAt, 'h264', mediaPtsMicros);361+ commitVideoFrame(formatPlan, keyframe, enqueued);501362 }502363 }503364
@@ -527,12 +388,8 @@ export function createVideoPipelineSession({
527388 avSkewPercentiles: avSkew.summary(at),528389 decoderQueueDepth: videoDecoder?.decodeQueueSize ?? 0,529390 decoderQueueCapacity: MAX_DECODE_QUEUE_SIZE,530- presenterQueueDepth: hasWebCodecs531- ? framePresenter.depth532- : Number(activeJpegDecode !== null) + Number(queuedJpegFrame !== null),533- presenterQueueCapacity: hasWebCodecs534- ? PRESENTER_QUEUE_CAPACITY535- : JPEG_PRESENTER_QUEUE_CAPACITY,391+ presenterQueueDepth: framePresenter.depth,392+ presenterQueueCapacity: PRESENTER_QUEUE_CAPACITY,536393 recovering: decoderRecovery.recovering,537394 };538395 }
M portal/video-pipeline.test.mjs +1 -229
@@ -18,7 +18,7 @@ const h264Delta = () => new Uint8Array([
1818 0, 0, 0, 1, 0x41, 0x9a,1919 ]);2020 21-function harness({ hasWebCodecs = true } = {}) {21+function harness() {2222 let clock = 10;2323 const decoders = [];2424 const chunks = [];
@@ -26,11 +26,6 @@ function harness({ hasWebCodecs = true } = {}) {
2626 const audioResets = [];2727 const formats = [];2828 const draws = [];29- const revokedUrls = [];30- const images = [];31- const imageStarts = [];32- const abortedImages = [];33- let nextObjectUrl = 0;3429 const canvas = { width: 0, height: 0 };3530 const context = {3631 drawImage: (...args) => draws.push(args),
@@ -52,7 +47,6 @@ function harness({ hasWebCodecs = true } = {}) {
5247 }5348 5449 const pipeline = createVideoPipelineSession({55- hasWebCodecs,5650 canvas,5751 context,5852 requestKeyframe: (reason) => keyframeRequests.push(reason),
@@ -66,28 +60,6 @@ function harness({ hasWebCodecs = true } = {}) {
6660 chunks.push(chunk);6761 return chunk;6862 },69- decodeBase64: () => new Uint8Array([1, 2, 3]),70- createBlob: (parts, options) => ({ parts, options }),71- createObjectUrl: () => `blob:test-frame-${++nextObjectUrl}`,72- revokeObjectUrl: (url) => revokedUrls.push(url),73- createImage: () => {74- let src = null;75- const image = {76- onload: null,77- onerror: null,78- get src() { return src; },79- set src(value) {80- src = value;81- imageStarts.push({ image: this, value });82- },83- };84- images.push(image);85- return image;86- },87- abortImage: (image) => {88- abortedImages.push(image);89- image.src = null;90- },9163 requestFrame: () => 1,9264 cancelFrame: () => {},9365 setTimer: () => 2,
@@ -103,10 +75,6 @@ function harness({ hasWebCodecs = true } = {}) {
10375 audioResets,10476 formats,10577 draws,106- revokedUrls,107- images,108- imageStarts,109- abortedImages,11078 advance(milliseconds = 1) { clock += milliseconds; },11179 };11280 }
@@ -121,7 +89,6 @@ function processH264Keyframe(subject, overrides = {}) {
12189 sequence: 1,12290 pts_micros: 1_000,12391 discontinuity: false,124- data: null,12592 ...overrides,12693 }, h264Keyframe());12794 }
@@ -169,7 +136,6 @@ test('unsupported codecs are dropped before decoder construction', () => {
169136 sequence: 1,170137 pts_micros: 2_000,171138 discontinuity: false,172- data: null,173139 }, h264Keyframe());174140 175141 assert.equal(subject.decoders.length, 0);
@@ -210,7 +176,6 @@ test('decode queue overload enters one bounded recovery episode and drops deltas
210176 codecConfig: false,211177 sequence: 2,212178 pts_micros: 2_000,213- data: null,214179 }, h264Delta());215180 216181 assert.deepEqual(subject.keyframeRequests, ['frontend-backpressure']);
@@ -223,199 +188,6 @@ test('decode queue overload enters one bounded recovery episode and drops deltas
223188 assert.equal(stats.decoderQueueDepth, 0);224189 });225190 226-test('JPEG fallback keeps its asynchronous base64 draw behavior', () => {227- const subject = harness({ hasWebCodecs: false });228- subject.pipeline.processFramePayload({229- width: 640,230- height: 400,231- data: 'AQID',232- keyframe: true,233- }, new Uint8Array([9, 9, 9]));234-235- assert.deepEqual(subject.pipeline.format, {236- codec: 'h264', width: 640, height: 400, epoch: 0,237- });238- assert.deepEqual(subject.formats, [subject.pipeline.format]);239- assert.equal(subject.canvas.width, 640);240- assert.equal(subject.canvas.height, 400);241- assert.equal(subject.images[0].src, 'blob:test-frame-1');242- assert.deepEqual(subject.draws, []);243- assert.deepEqual(subject.revokedUrls, []);244- assert.equal(subject.pipeline.snapshot().presenterQueueDepth, 1);245- assert.equal(subject.pipeline.snapshot().presenterQueueCapacity, 2);246-247- subject.advance();248- subject.images[0].onload();249- assert.equal(subject.draws.length, 1);250- assert.deepEqual(subject.revokedUrls, ['blob:test-frame-1']);251- assert.equal(subject.pipeline.snapshot().presentedFrames, 1);252- assert.equal(subject.pipeline.snapshot().presenterQueueDepth, 0);253- assert.equal(subject.pipeline.snapshot().presenterQueueCapacity, 2);254-});255-256-test('JPEG fallback suppresses an active stale decode before starting the newest frame', () => {257- const subject = harness({ hasWebCodecs: false });258- const payload = {259- width: 640,260- height: 400,261- data: 'AQID',262- keyframe: true,263- };264- subject.pipeline.processFramePayload(payload);265- const staleLoad = subject.images[0].onload;266- const staleError = subject.images[0].onerror;267-268- subject.advance(10);269- subject.pipeline.processFramePayload(payload);270- assert.equal(subject.images.length, 1);271- assert.deepEqual(subject.revokedUrls, []);272- const afterOverwrite = subject.pipeline.snapshot();273- assert.equal(afterOverwrite.presentationDroppedFrames, 1);274- assert.equal(afterOverwrite.presenterQueueDepth, 2);275- assert.equal(afterOverwrite.presenterQueueCapacity, 2);276-277- subject.advance(5);278- staleLoad();279- assert.deepEqual(subject.draws, []);280- assert.deepEqual(subject.revokedUrls, ['blob:test-frame-1']);281- assert.equal(subject.images.length, 2);282- assert.equal(subject.images[1].src, 'blob:test-frame-2');283- const newestLoad = subject.images[1].onload;284- assert.equal(subject.pipeline.snapshot().presenterQueueDepth, 1);285-286- subject.advance(5);287- newestLoad();288- staleError(new Error('stale JPEG failure'));289- const afterNewest = subject.pipeline.snapshot();290- assert.deepEqual(subject.draws.map(([image]) => image), [subject.images[1]]);291- assert.equal(afterNewest.presentedFrames, 1);292- assert.equal(afterNewest.presentPercentiles.count, 1);293- assert.equal(afterNewest.presentPercentiles.p50, 10);294- assert.equal(afterNewest.presentationDroppedFrames, 1);295- assert.equal(afterNewest.droppedFrames, 0);296- assert.equal(afterNewest.presenterQueueDepth, 0);297- assert.equal(afterNewest.presenterQueueCapacity, 2);298- assert.deepEqual(subject.revokedUrls, [299- 'blob:test-frame-1',300- 'blob:test-frame-2',301- ]);302-});303-304-test('JPEG fallback error callbacks affect only the current pending frame', () => {305- const subject = harness({ hasWebCodecs: false });306- const payload = {307- width: 640,308- height: 400,309- data: 'AQID',310- keyframe: true,311- };312- subject.pipeline.processFramePayload(payload);313- const staleError = subject.images[0].onerror;314-315- subject.pipeline.processFramePayload(payload);316- staleError(new Error('stale JPEG failure'));317- assert.equal(subject.pipeline.snapshot().droppedFrames, 0);318- assert.equal(subject.images.length, 2);319- const failedLoad = subject.images[1].onload;320- const currentError = subject.images[1].onerror;321-322- currentError(new Error('current JPEG failure'));323- failedLoad();324- const afterFailure = subject.pipeline.snapshot();325- assert.equal(afterFailure.droppedFrames, 1);326- assert.equal(afterFailure.presentedFrames, 0);327- assert.equal(afterFailure.presentationDroppedFrames, 1);328- assert.equal(afterFailure.presenterQueueDepth, 0);329- assert.equal(afterFailure.presenterQueueCapacity, 2);330- assert.deepEqual(subject.draws, []);331- assert.deepEqual(subject.revokedUrls, [332- 'blob:test-frame-1',333- 'blob:test-frame-2',334- ]);335-});336-337-test('JPEG fallback callbacks cannot survive a session reset', () => {338- const subject = harness({ hasWebCodecs: false });339- subject.pipeline.processFramePayload({340- width: 640,341- height: 400,342- data: 'AQID',343- keyframe: true,344- });345- const staleLoad = subject.images[0].onload;346- const staleError = subject.images[0].onerror;347-348- subject.pipeline.reset();349- subject.advance();350- staleLoad();351- staleError(new Error('reset JPEG failure'));352-353- const stats = subject.pipeline.snapshot();354- assert.equal(stats.receivedFrames, 0);355- assert.equal(stats.presentedFrames, 0);356- assert.equal(stats.droppedFrames, 0);357- assert.equal(stats.presentPercentiles.count, 0);358- assert.equal(stats.presenterQueueDepth, 0);359- assert.equal(stats.presenterQueueCapacity, 2);360- assert.deepEqual(subject.draws, []);361- assert.deepEqual(subject.revokedUrls, ['blob:test-frame-1']);362- assert.deepEqual(subject.abortedImages, [subject.images[0]]);363-});364-365-test('JPEG fallback bounds delayed decoding to one active and one coalesced latest payload', () => {366- const subject = harness({ hasWebCodecs: false });367- const payload = {368- width: 640,369- height: 400,370- data: 'AQID',371- keyframe: true,372- };373- subject.pipeline.processFramePayload(payload);374- const oldestLoad = subject.images[0].onload;375-376- for (let frame = 2; frame <= 10; frame++) {377- subject.advance();378- subject.pipeline.processFramePayload(payload);379- }380-381- assert.equal(subject.images.length, 1);382- assert.deepEqual(subject.imageStarts, [{383- image: subject.images[0],384- value: 'blob:test-frame-1',385- }]);386- assert.deepEqual(subject.revokedUrls, []);387- let stats = subject.pipeline.snapshot();388- assert.equal(stats.presentationDroppedFrames, 9);389- assert.equal(stats.presenterQueueDepth, 2);390- assert.equal(stats.presenterQueueCapacity, 2);391-392- subject.advance();393- oldestLoad();394- assert.equal(subject.images.length, 2);395- assert.deepEqual(subject.imageStarts, [396- { image: subject.images[0], value: 'blob:test-frame-1' },397- { image: subject.images[1], value: 'blob:test-frame-2' },398- ]);399- assert.deepEqual(subject.revokedUrls, ['blob:test-frame-1']);400- stats = subject.pipeline.snapshot();401- assert.equal(stats.presenterQueueDepth, 1);402- assert.equal(stats.presenterQueueCapacity, 2);403-404- subject.advance();405- subject.images[1].onload();406- stats = subject.pipeline.snapshot();407- assert.deepEqual(subject.draws.map(([image]) => image), [subject.images[1]]);408- assert.equal(stats.presentedFrames, 1);409- assert.equal(stats.presentPercentiles.p50, 2);410- assert.equal(stats.presentationDroppedFrames, 9);411- assert.equal(stats.presenterQueueDepth, 0);412- assert.equal(stats.presenterQueueCapacity, 2);413- assert.deepEqual(subject.revokedUrls, [414- 'blob:test-frame-1',415- 'blob:test-frame-2',416- ]);417-});418-419191 test('reset closes decoding state but preserves the last dimensions and codec quirk', () => {420192 const subject = harness();421193 processH264Keyframe(subject);
M src-tauri/Cargo.toml +0 -3
M src-tauri/src/commands/network.rs +1 -4
M src-tauri/src/commands/state.rs +10 -116
@@ -193,7 +193,6 @@ fn reapply_hidden_cursor_rect(
193193 set_visible(false)194194 }195195 196-pub const FRAME_ALPN: &[u8] = sigil_protocol::LEGACY_FRAME_ALPN_V0;197196 pub const AUDIO_DELIVERY_CAPACITY: usize = 3;198197 199198 #[derive(Debug, Default)]
@@ -280,21 +279,12 @@ pub const fn development_direct_node_available() -> bool {
280279 cfg!(any(debug_assertions, feature = "demo-direct-node"))281280 }282281 283-pub const fn development_force_jpeg_available() -> bool {284- cfg!(debug_assertions)285-}286-287282 #[derive(Clone, Debug, Default, PartialEq, Eq)]288283 pub struct LaunchOptions {289284 pub dev_connect_node_id: Option<iroh::PublicKey>,290- pub dev_force_jpeg: bool,291285 }292286 293-pub fn parse_launch_options<I, S>(294- args: I,295- allow_dev_connect: bool,296- allow_force_jpeg: bool,297-) -> Result<LaunchOptions, String>287+pub fn parse_launch_options<I, S>(args: I, allow_dev_connect: bool) -> Result<LaunchOptions, String>298288 where299289 I: IntoIterator<Item = S>,300290 S: Into<OsString>,
@@ -316,19 +306,6 @@ where
316306 let arg = arg317307 .into_string()318308 .map_err(|_| "Portal command-line arguments must be valid UTF-8".to_string())?;319- if arg == "--dev-force-jpeg" {320- if !allow_force_jpeg {321- return Err(322- "--dev-force-jpeg requires a debug build; ordinary release builds use the normal WebCodecs capability probe"323- .to_string(),324- );325- }326- if options.dev_force_jpeg {327- return Err("--dev-force-jpeg may be specified only once".to_string());328- }329- options.dev_force_jpeg = true;330- continue;331- }332309 let node_id = if arg == "--dev-connect" {333310 Some(334311 args.next()
@@ -382,7 +359,6 @@ pub struct AppState {
382359 pub client_connection_active: Arc<AtomicBool>,383360 pub webcodecs: AtomicBool,384361 pub dev_connect_node_id: Option<iroh::PublicKey>,385- pub dev_force_jpeg: bool,386362 }387363 388364 pub type MediaControlRequestSender =
@@ -484,12 +460,7 @@ impl AppState {
484460 I: IntoIterator<Item = S>,485461 S: Into<OsString>,486462 {487- parse_launch_options(488- args,489- development_direct_node_available(),490- development_force_jpeg_available(),491- )492- .map(Self::new)463+ parse_launch_options(args, development_direct_node_available()).map(Self::new)493464 }494465 495466 fn new(options: LaunchOptions) -> Self {
@@ -510,7 +481,6 @@ impl AppState {
510481 client_connection_active: Arc::new(AtomicBool::new(false)),511482 webcodecs: AtomicBool::new(false),512483 dev_connect_node_id: options.dev_connect_node_id,513- dev_force_jpeg: options.dev_force_jpeg,514484 }515485 }516486 }
@@ -522,8 +492,6 @@ pub struct DevelopmentConnectionMode {
522492 pub enabled: bool,523493 pub host_node_id: Option<String>,524494 pub warning: &'static str,525- pub force_jpeg: bool,526- pub jpeg_warning: &'static str,527495 }528496 529497 #[tauri::command]
@@ -532,8 +500,6 @@ pub fn development_connection_mode(state: tauri::State<'_, AppState>) -> Develop
532500 enabled: state.dev_connect_node_id.is_some(),533501 host_node_id: state.dev_connect_node_id.map(|node_id| node_id.to_string()),534502 warning: "Development direct-node routing skips passkey identity lookup; it is not client authorization.",535- force_jpeg: state.dev_force_jpeg,536- jpeg_warning: "Development JPEG compatibility mode is forced; WebCodecs remains probed but is not selected.",537503 }538504 }539505
@@ -571,15 +537,10 @@ pub fn set_client_cursor_grab(window: tauri::WebviewWindow, grab: bool) -> Resul
571537 .map(|()| !cfg!(target_os = "macos"))572538 }573539 574-const fn effective_webcodecs_available(available: bool, force_jpeg: bool) -> bool {575- available && !force_jpeg576-}577-578540 #[tauri::command]579541 pub fn set_webcodecs_available(state: tauri::State<'_, AppState>, available: bool) -> bool {580- let effective = effective_webcodecs_available(available, state.dev_force_jpeg);581- state.webcodecs.store(effective, Ordering::SeqCst);582- effective542+ state.webcodecs.store(available, Ordering::SeqCst);543+ available583544 }584545 585546 #[tauri::command]
@@ -688,41 +649,14 @@ mod tests {
688649 #[test]689650 fn parses_debug_direct_node_flag() {690651 let node_id = test_node_id();691- let options =692- parse_launch_options(["portal", "--dev-connect", &node_id], true, true).unwrap();652+ let options = parse_launch_options(["portal", "--dev-connect", &node_id], true).unwrap();693653 694654 assert_eq!(options.dev_connect_node_id.unwrap().to_string(), node_id);695655 }696656 697- #[test]698- fn parses_debug_force_jpeg_flag() {699- let options = parse_launch_options(["portal", "--dev-force-jpeg"], true, true).unwrap();700-701- assert!(options.dev_force_jpeg);702- }703-704- #[test]705- fn parses_combined_debug_hardware_proof_flags() {706- let node_id = test_node_id();707- let options = parse_launch_options(708- [709- "portal",710- "--dev-connect",711- node_id.as_str(),712- "--dev-force-jpeg",713- ],714- true,715- true,716- )717- .unwrap();718-719- assert_eq!(options.dev_connect_node_id.unwrap().to_string(), node_id);720- assert!(options.dev_force_jpeg);721- }722-723657 #[test]724658 fn rejects_legacy_daemon_mode() {725- let error = parse_launch_options(["portal", "--daemon"], true, true).unwrap_err();659+ let error = parse_launch_options(["portal", "--daemon"], true).unwrap_err();726660 assert!(error.contains("separate sigil executable"));727661 }728662
@@ -730,7 +664,7 @@ mod tests {
730664 fn parses_equals_form() {731665 let node_id = test_node_id();732666 let flag = format!("--dev-connect={node_id}");733- let options = parse_launch_options(["portal", &flag], true, true).unwrap();667+ let options = parse_launch_options(["portal", &flag], true).unwrap();734668 735669 assert_eq!(options.dev_connect_node_id.unwrap().to_string(), node_id);736670 }
@@ -738,19 +672,11 @@ mod tests {
738672 #[test]739673 fn rejects_direct_node_when_debug_mode_is_disabled() {740674 let node_id = test_node_id();741- let error =742- parse_launch_options(["portal", "--dev-connect", &node_id], false, true).unwrap_err();675+ let error = parse_launch_options(["portal", "--dev-connect", &node_id], false).unwrap_err();743676 744677 assert!(error.contains("debug build or the explicit demo-direct-node feature"));745678 }746679 747- #[test]748- fn rejects_force_jpeg_when_debug_mode_is_disabled() {749- let error = parse_launch_options(["portal", "--dev-force-jpeg"], true, false).unwrap_err();750-751- assert!(error.contains("requires a debug build"));752- }753-754680 #[test]755681 fn ordinary_release_excludes_direct_node_bypass() {756682 if cfg!(not(debug_assertions)) && cfg!(not(feature = "demo-direct-node")) {
@@ -766,28 +692,6 @@ mod tests {
766692 assert_eq!(result.is_ok(), development_direct_node_available());767693 }768694 769- #[test]770- fn ordinary_release_excludes_force_jpeg_override() {771- if cfg!(not(debug_assertions)) {772- assert!(!development_force_jpeg_available());773- }774- }775-776- #[test]777- fn app_state_accepts_force_jpeg_only_in_debug_builds() {778- let result = AppState::from_args(["portal", "--dev-force-jpeg"]);779-780- assert_eq!(result.is_ok(), development_force_jpeg_available());781- }782-783- #[test]784- fn forced_jpeg_publication_keeps_native_and_frontend_mode_aligned() {785- assert!(effective_webcodecs_available(true, false));786- assert!(!effective_webcodecs_available(true, true));787- assert!(!effective_webcodecs_available(false, false));788- assert!(!effective_webcodecs_available(false, true));789- }790-791695 #[test]792696 fn app_state_starts_without_owned_transport_connections() {793697 let state = AppState::default();
@@ -802,12 +706,12 @@ mod tests {
802706 #[test]803707 fn rejects_missing_invalid_and_duplicate_node_ids() {804708 assert!(805- parse_launch_options(["portal", "--dev-connect"], true, true)709+ parse_launch_options(["portal", "--dev-connect"], true)806710 .unwrap_err()807711 .contains("requires an iroh node ID")808712 );809713 assert!(810- parse_launch_options(["portal", "--dev-connect", "not-a-node-id"], true, true)714+ parse_launch_options(["portal", "--dev-connect", "not-a-node-id"], true)811715 .unwrap_err()812716 .contains("Invalid iroh node ID")813717 );
@@ -823,16 +727,6 @@ mod tests {
823727 &node_id,824728 ],825729 true,826- true,827- )828- .unwrap_err()829- .contains("only once")830- );831- assert!(832- parse_launch_options(833- ["portal", "--dev-force-jpeg", "--dev-force-jpeg"],834- true,835- true,836730 )837731 .unwrap_err()838732 .contains("only once")
M src-tauri/src/media/audio_delivery.rs +2 -4
M src-tauri/src/media/connect.rs +51 -98
@@ -14,9 +14,7 @@ use tauri::{
1414 1515 use crate::commands::auth::derive_iroh_secret_from_key;1616 use crate::commands::enrollment::{connection_enrollment, mark_invitation_redeemed};17-use crate::commands::state::{18- AppState, FRAME_ALPN, MediaFeedbackSender, development_direct_node_available,19-};17+use crate::commands::state::{AppState, MediaFeedbackSender, development_direct_node_available};2018 use crate::media::adaptive_feedback::{2119 CLIENT_MEDIA_FEEDBACK_CONNECT_TIMEOUT, open_negotiated_feedback_stream,2220 run_media_feedback_session,
@@ -103,6 +101,16 @@ pub(crate) async fn connect_client(
103101 })?;104102 let connect_guard = ClientConnectGuard::acquire(Arc::clone(&state.client_connection_active))?;105103 104+ // Binary WebCodecs delivery is the only video path; fail before any105+ // network work so a webview without H.264 WebCodecs support surfaces a106+ // clear error instead of a black stream.107+ if !state.webcodecs.load(Ordering::SeqCst) {108+ return Err(109+ "WebCodecs H.264 decoding is unavailable in this webview; Portal cannot stream video"110+ .to_string(),111+ );112+ }113+106114 let (client_secret, development_mode) = if let Some(node_id) = state.dev_connect_node_id {107115 if !development_direct_node_available() {108116 return Err(
@@ -165,67 +173,37 @@ pub(crate) async fn connect_client(
165173 // routing and fallback across all N0 relays automatically.166174 let addr = iroh::EndpointAddr::new(host_node_id);167175 168- // Public Sigil authorization exists only on the bounded, negotiated v1169- // protocols. The inherited v0 leg is retained below solely as migration170- // code and is no longer selected by an ordinary Portal connection.171- let use_v1 = true;172- let (frame_conn, frame_recv, media_control_stream, media_negotiation, media_transport) =173- if use_v1 {174- let first_attempt = open_negotiated_media_stream(175- &endpoint,176- &addr,177- handshake_nonce,178- invitation.as_deref(),179- )176+ let first_attempt =177+ open_negotiated_media_stream(&endpoint, &addr, handshake_nonce, invitation.as_deref())180178 .await;181- let (connection, recv, control, negotiation, transport) = match first_attempt {182- Ok(result) => result,183- Err(invitation_error)184- if invitation.is_some()185- && invitation_error.contains("Portal peer is not authorized") =>186- {187- // Recover only the narrow crash window where Sigil durably188- // consumed the invitation but Portal did not durably clear it.189- // The replay itself remains rejected; a second, ticket-free190- // connection can succeed only as the already-enrolled Iroh191- // peer authenticated by the exact invited host.192- open_negotiated_media_stream(&endpoint, &addr, handshake_nonce, None)179+ let (frame_conn, frame_recv, media_control_stream, media_negotiation, media_transport) =180+ match first_attempt {181+ Ok(result) => result,182+ Err(invitation_error)183+ if invitation.is_some()184+ && invitation_error.contains("Portal peer is not authorized") =>185+ {186+ // Recover only the narrow crash window where Sigil durably187+ // consumed the invitation but Portal did not durably clear it.188+ // The replay itself remains rejected; a second, ticket-free189+ // connection can succeed only as the already-enrolled Iroh190+ // peer authenticated by the exact invited host.191+ open_negotiated_media_stream(&endpoint, &addr, handshake_nonce, None)193192 .await194193 .map_err(|retry_error| {195194 format!(196195 "{invitation_error}; ticket-free enrollment recovery also failed: {retry_error}"197196 )198197 })?199- }200- Err(error) => return Err(error),201- };202- (connection, recv, control, Some(negotiation), transport)203- } else {204- let connection = endpoint205- .connect(addr.clone(), FRAME_ALPN)206- .await207- .map_err(|e| format!("Failed to connect frame stream: {e}"))?;208- let (mut send, recv) = connection209- .open_bi()210- .await211- .map_err(|e| format!("Failed to open frame stream: {e}"))?;212- send.write_all(&[1u8])213- .await214- .map_err(|e| format!("Failed to send start: {e}"))?;215- send.finish()216- .map_err(|e| format!("Failed to finish frame start stream: {e}"))?;217- (connection, recv, None, None, MediaTransport::LegacyV0)198+ }199+ Err(error) => return Err(error),218200 };219- let media_session_id = media_negotiation220- .as_ref()221- .map(|negotiation| negotiation.session_id);201+ let media_session_id = media_negotiation.session_id;222202 let (upstream_moq_media, frame_connection_for_stats) = if media_transport223203 == MediaTransport::UpstreamMoq224204 {225- let session_id = media_session_id226- .ok_or_else(|| "Host omitted the control session ID required by MoQ".to_string())?;227205 let (receiver, diagnostics_connection) =228- match open_upstream_moq_media(&endpoint, &addr, session_id).await {206+ match open_upstream_moq_media(&endpoint, &addr, media_session_id).await {229207 Ok(media) => media,230208 Err(error) => {231209 // CONTROL already authenticated and owns the host's
@@ -242,9 +220,7 @@ pub(crate) async fn connect_client(
242220 } else {243221 (None, frame_conn.clone())244222 };245- let pointer_surface_dimensions = media_negotiation246- .as_ref()247- .and_then(|negotiation| negotiation.pointer_surface_dimensions);223+ let pointer_surface_dimensions = media_negotiation.pointer_surface_dimensions;248224 if !development_mode && let Some(expected_invitation) = invitation.as_deref() {249225 // An accepted media hello means Sigil durably committed the one-time250226 // enrollment before returning. Future PIN/tap/play sessions send no
@@ -256,33 +232,22 @@ pub(crate) async fn connect_client(
256232 // and the grouped-v3 compatibility path. Unsupported ALPN is normal257233 // compatibility with older Sigil hosts; all other failures remain visible258234 // diagnostics but never downgrade the authenticated media session.259- let (adaptive_feedback_stream, adaptive_feedback_error) = if media_transport260- .supports_adaptive_feedback()235+ let (adaptive_feedback_stream, adaptive_feedback_error) = match tokio::time::timeout(236+ CLIENT_MEDIA_FEEDBACK_CONNECT_TIMEOUT,237+ open_negotiated_feedback_stream(&endpoint, &addr, handshake_nonce, media_session_id),238+ )239+ .await261240 {262- match tokio::time::timeout(263- CLIENT_MEDIA_FEEDBACK_CONNECT_TIMEOUT,264- open_negotiated_feedback_stream(265- &endpoint,266- &addr,267- handshake_nonce,268- media_session_id.ok_or_else(|| "Media v3 omitted its session ID".to_string())?,269- ),270- )271- .await272- {273- Ok(Ok(stream)) => (stream, None),274- Ok(Err(error)) => {275- eprintln!("[client] adaptive feedback unavailable: {error}");276- (None, Some(error))277- }278- Err(_) => {279- let error = "Adaptive feedback negotiation timed out".to_string();280- eprintln!("[client] adaptive feedback unavailable: {error}");281- (None, Some(error))282- }241+ Ok(Ok(stream)) => (stream, None),242+ Ok(Err(error)) => {243+ eprintln!("[client] adaptive feedback unavailable: {error}");244+ (None, Some(error))245+ }246+ Err(_) => {247+ let error = "Adaptive feedback negotiation timed out".to_string();248+ eprintln!("[client] adaptive feedback unavailable: {error}");249+ (None, Some(error))283250 }284- } else {285- (None, None)286251 };287252 288253 let InputSession {
@@ -291,15 +256,7 @@ pub(crate) async fn connect_client(
291256 recv: input_recv,292257 capabilities: input_capabilities,293258 availability: input_availability,294- } = open_input_session(295- &endpoint,296- &addr,297- handshake_nonce,298- media_session_id,299- grants,300- use_v1,301- )302- .await?;259+ } = open_input_session(&endpoint, &addr, handshake_nonce, media_session_id, grants).await?;303260 304261 let network_diagnostics = Arc::new(StdMutex::new(NetworkSessionDiagnostics::new(305262 Instant::now(),
@@ -379,17 +336,17 @@ pub(crate) async fn connect_client(
379336 } else {380337 *state.media_feedback.lock().await = None;381338 }382- let media_control_requests = if let Some(control_stream) = media_control_stream {339+ let media_control_requests = {383340 let (control_tx, control_rx) = tokio::sync::mpsc::channel(1);384341 *state.media_control.lock().await = Some((media_generation, control_tx.clone()));385- tokio::spawn(run_media_control_writer_v3(control_stream, control_rx));342+ tokio::spawn(run_media_control_writer_v3(343+ media_control_stream,344+ control_rx,345+ ));386346 if media_transport == MediaTransport::GroupedObjectsV3 {387347 let _ = control_tx.try_send((KeyframeRequestReasonV3::Join, None));388348 }389349 Some(control_tx)390- } else {391- *state.media_control.lock().await = None;392- None393350 };394351 let frame_events_in_flight = Arc::new(AtomicUsize::new(0));395352 *state.frame_delivery.lock().await =
@@ -402,13 +359,10 @@ pub(crate) async fn connect_client(
402359 tokio::spawn(run_input_forwarder(403360 input_send,404361 rx,405- use_v1,406362 input_capabilities,407363 input_send_diagnostics,408364 ));409365 410- // Frame reader — dual path: WebCodecs (raw bytes) or software JPEG decode411- let use_webcodecs = state.webcodecs.load(Ordering::SeqCst);412366 tokio::spawn(run_video_delivery(VideoDeliveryRequest {413367 app,414368 endpoint,
@@ -424,7 +378,6 @@ pub(crate) async fn connect_client(
424378 media_transport,425379 media_generation,426380 connected_audio_generation,427- use_webcodecs,428381 }));429382 430383 let result = ConnectResult {
M src-tauri/src/media/frame_channel.rs +0 -44
@@ -6,28 +6,6 @@ use tauri::{AppHandle, Emitter};
66 77 use crate::commands::state::AppState;88 9-pub(crate) fn byte_to_codec(value: u8) -> Result<&'static str, String> {10- match value {11- 1 => Ok("h264"),12- unsupported => Err(format!(13- "Unsupported legacy media codec: {unsupported}; Portal supports H.264 only"14- )),15- }16-}17-18-#[derive(Serialize, Clone)]19-pub struct FramePayload {20- pub generation: u64,21- pub width: u32,22- pub height: u32,23- pub data: String,24- pub keyframe: bool,25- pub codec: String,26- pub capture_timestamp_micros: Option<u64>,27- pub pts_micros: Option<i64>,28- pub discontinuity: bool,29-}30-319 #[derive(Serialize, Clone)]3210 struct FrameErrorPayload {3311 generation: u64,
@@ -419,15 +397,6 @@ mod tests {
419397 );420398 }421399 422- #[test]423- fn legacy_codec_bytes_fail_closed_to_h264_only() {424- assert_eq!(byte_to_codec(1).unwrap(), "h264");425- for unsupported in [0, 2, 3, u8::MAX] {426- let error = byte_to_codec(unsupported).unwrap_err();427- assert!(error.contains("H.264 only"));428- }429- }430-431400 #[test]432401 fn frame_channel_slots_are_bounded_and_cannot_underflow() {433402 let in_flight = AtomicUsize::new(0);
@@ -472,25 +441,12 @@ mod tests {
472441 473442 #[test]474443 fn global_frame_events_serialize_their_media_generation() {475- let frame = serde_json::to_value(FramePayload {476- generation: 17,477- width: 1280,478- height: 800,479- data: "jpeg".to_string(),480- keyframe: true,481- codec: "h264".to_string(),482- capture_timestamp_micros: Some(1),483- pts_micros: Some(2),484- discontinuity: false,485- })486- .unwrap();487444 let error = serde_json::to_value(FrameErrorPayload {488445 generation: 17,489446 error: "closed".to_string(),490447 })491448 .unwrap();492449 493- assert_eq!(frame["generation"], 17);494450 assert_eq!(error["generation"], 17);495451 assert_eq!(error["error"], "closed");496452 }
M src-tauri/src/media/input_delivery.rs +27 -79
@@ -14,7 +14,6 @@ use crate::media::network_diagnostics::{NetworkSessionDiagnostics, lock_network_
1414 use crate::media::transport::{NegotiatedV1Stream, negotiate_v1};1515 use crate::platform_capabilities::relative_pointer_capture_enabled;1616 17-const INPUT_ALPN: &[u8] = sigil_protocol::LEGACY_INPUT_ALPN_V0;1817 pub(crate) const CLIENT_INPUT_QUEUE_CAPACITY: usize = 256;1918 2019 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -271,23 +270,14 @@ fn input_event_allowed(capabilities: &[Capability], event: &InputEvent) -> bool
271270 async fn write_client_input_event(272271 stream: &mut iroh::endpoint::SendStream,273272 event: &InputEvent,274- use_v1: bool,275273 diagnostics: Option<&Arc<StdMutex<NetworkSessionDiagnostics>>>,276274 ) -> Result<(), String> {277275 if let Some(diagnostics) = diagnostics {278276 lock_network_diagnostics(diagnostics).begin_input_send(Instant::now());279277 }280- if use_v1 {281- write_input_event(stream, event)282- .await283- .map_err(|error| error.to_string())284- } else {285- let json = serde_json::to_string(event).map_err(|error| error.to_string())?;286- stream287- .write_all(format!("{json}\n").as_bytes())288- .await289- .map_err(|error| error.to_string())290- }278+ write_input_event(stream, event)279+ .await280+ .map_err(|error| error.to_string())291281 }292282 293283 fn observe_input_ack_if_negotiated(
@@ -314,58 +304,30 @@ pub(crate) async fn open_input_session(
314304 endpoint: &Endpoint,315305 address: &iroh::EndpointAddr,316306 nonce: [u8; 16],317- media_session_id: Option<u64>,307+ media_session_id: u64,318308 grants: InvitationGrants,319- use_v1: bool,320309 ) -> Result<InputSession, String> {321- let input_alpn = if use_v1 { INPUT_ALPN_V1 } else { INPUT_ALPN };322- let (connection, send, recv, capabilities) = if use_v1 {323- let mut errors = Vec::new();324- let mut accepted = None;325- // Older hosts reject unknown capability enum values. Try all four326- // pointer feature levels with ACK first, then repeat the exact legacy327- // offers without ACK so lack of diagnostics never forces absolute328- // pointer input prematurely.329- for capabilities in input_capability_offers(grants) {330- match open_negotiated_input_stream(endpoint, address, nonce, capabilities).await {331- Ok(result) => {332- accepted = Some(result);333- break;334- }335- Err(error) => errors.push(error),310+ let mut errors = Vec::new();311+ let mut accepted = None;312+ // Older hosts reject unknown capability enum values. Try all four313+ // pointer feature levels with ACK first, then repeat the exact legacy314+ // offers without ACK so lack of diagnostics never forces absolute315+ // pointer input prematurely.316+ for capabilities in input_capability_offers(grants) {317+ match open_negotiated_input_stream(endpoint, address, nonce, capabilities).await {318+ Ok(result) => {319+ accepted = Some(result);320+ break;336321 }322+ Err(error) => errors.push(error),337323 }338- let (connection, send, recv, input_negotiation) = accepted339- .ok_or_else(|| format!("All input capability offers failed: {}", errors.join("; ")))?;340- if Some(input_negotiation.session_id) != media_session_id {341- return Err("Host returned mismatched media and input sessions".to_string());342- }343- (connection, send, recv, input_negotiation.capabilities)344- } else {345- let input_conn = endpoint346- .connect(address.clone(), input_alpn)347- .await348- .map_err(|e| format!("Failed to connect input stream: {}", e))?;349- let (mut send, recv) = input_conn350- .open_bi()351- .await352- .map_err(|e| format!("Failed to open input stream: {}", e))?;353- send.write_all(&[1u8])354- .await355- .map_err(|e| format!("Failed to send input start: {}", e))?;356- // The inherited protocol predates negotiation and supports the current357- // absolute-pointer, keyboard, and text event set.358- (359- input_conn,360- send,361- recv,362- vec![363- Capability::AbsolutePointer,364- Capability::Keyboard,365- Capability::Text,366- ],367- )368- };324+ }325+ let (connection, send, recv, input_negotiation) = accepted326+ .ok_or_else(|| format!("All input capability offers failed: {}", errors.join("; ")))?;327+ if input_negotiation.session_id != media_session_id {328+ return Err("Host returned mismatched media and input sessions".to_string());329+ }330+ let capabilities = input_negotiation.capabilities;369331 // A malformed or future host must not be able to re-enable a local input370332 // path that this Portal build deliberately withheld from its offers.371333 let capabilities = mask_unavailable_relative_pointer_capabilities(
@@ -438,7 +400,6 @@ pub(crate) async fn run_input_feedback(
438400 pub(crate) async fn run_input_forwarder(439401 mut input_stream: iroh::endpoint::SendStream,440402 rx: tokio::sync::mpsc::Receiver<InputEvent>,441- use_v1: bool,442403 input_capabilities: Vec<Capability>,443404 input_send_diagnostics: Arc<StdMutex<NetworkSessionDiagnostics>>,444405 ) {
@@ -460,7 +421,6 @@ pub(crate) async fn run_input_forwarder(
460421 if let Err(error) = write_client_input_event(461422 &mut input_stream,462423 &event,463- use_v1,464424 Some(&input_send_diagnostics),465425 )466426 .await
@@ -480,7 +440,6 @@ pub(crate) async fn run_input_forwarder(
480440 if let Err(error) = write_client_input_event(481441 &mut input_stream,482442 &event,483- use_v1,484443 Some(&input_send_diagnostics),485444 ).await {486445 eprintln!("[client] input stream write failed: {error}; disconnecting");
@@ -512,7 +471,6 @@ pub(crate) async fn run_input_forwarder(
512471 if let Err(error) = write_client_input_event(513472 &mut input_stream,514473 &relative_barrier,515- use_v1,516474 Some(&input_send_diagnostics),517475 )518476 .await
@@ -536,26 +494,16 @@ pub(crate) async fn run_input_forwarder(
536494 }537495 last_absolute_mouse_time = now;538496 }539- if let Err(error) = write_client_input_event(540- &mut input_stream,541- &event,542- use_v1,543- Some(&input_send_diagnostics),544- )545- .await497+ if let Err(error) =498+ write_client_input_event(&mut input_stream, &event, Some(&input_send_diagnostics)).await546499 {547500 eprintln!("[client] input stream write failed: {error}; disconnecting");548501 break;549502 }550503 }551504 while let Some(event) = pending_relative.take() {552- if let Err(error) = write_client_input_event(553- &mut input_stream,554- &event,555- use_v1,556- Some(&input_send_diagnostics),557- )558- .await505+ if let Err(error) =506+ write_client_input_event(&mut input_stream, &event, Some(&input_send_diagnostics)).await559507 {560508 eprintln!("[client] final relative input write failed: {error}");561509 break;
M src-tauri/src/media/object_receiver.rs +4 -406
@@ -1,254 +1,16 @@
11 use std::collections::BTreeMap;22 use std::time::Duration;33 4+#[cfg(test)]5+use sigil_protocol::MediaFrame;46 use sigil_protocol::{5- FrameFlags, KeyframeRequestReasonV3, MAX_MEDIA_GROUP_BYTES_V3, MediaFrame, MediaObjectV3,6- ProtocolError, read_media_object, read_media_object_v3,7+ FrameFlags, KeyframeRequestReasonV3, MAX_MEDIA_GROUP_BYTES_V3, MediaObjectV3, ProtocolError,8+ read_media_object_v3,79 };810 911 const CLIENT_MEDIA_OBJECT_CAPACITY: usize = 4;1012 const CLIENT_MEDIA_OBJECT_READ_TIMEOUT: Duration = Duration::from_secs(1);1113 12-#[derive(Debug)]13-pub(crate) enum MediaObjectReadOutcome {14- Frame {15- object_index: u64,16- frame: MediaFrame,17- },18- Dropped {19- object_index: u64,20- },21- Malformed(String),22-}23-24-impl MediaObjectReadOutcome {25- fn object_index(&self) -> Option<u64> {26- match self {27- Self::Frame { object_index, .. } | Self::Dropped { object_index } => {28- Some(*object_index)29- }30- Self::Malformed(_) => None,31- }32- }33-34- fn is_fast_forward_barrier(&self) -> bool {35- let Self::Frame { frame, .. } = self else {36- return false;37- };38- frame.header.flags.contains(FrameFlags::KEYFRAME)39- && frame.header.flags.contains(FrameFlags::CODEC_CONFIG)40- && frame.header.flags.contains(FrameFlags::DISCONTINUITY)41- }42-}43-44-#[derive(Debug)]45-struct MediaObjectReorder {46- next_object_index: u64,47- completed: BTreeMap<u64, MediaObjectReadOutcome>,48-}49-50-impl MediaObjectReorder {51- fn new(first_object_index: u64) -> Self {52- Self {53- next_object_index: first_object_index,54- completed: BTreeMap::new(),55- }56- }57-58- fn pending_len(&self) -> usize {59- self.completed.len()60- }61-62- fn push(63- &mut self,64- outcome: MediaObjectReadOutcome,65- ) -> Result<Option<MediaObjectReadOutcome>, String> {66- let Some(object_index) = outcome.object_index() else {67- // Malformed objects remain terminal as soon as their read completes.68- return Ok(Some(outcome));69- };70- if object_index < self.next_object_index {71- return Ok(Some(outcome));72- }73- if outcome.is_fast_forward_barrier() {74- self.completed75- .retain(|completed_index, _| *completed_index > object_index);76- self.next_object_index = object_index77- .checked_add(1)78- .ok_or_else(|| "Media object reorder index overflowed".to_string())?;79- return Ok(Some(outcome));80- }81- if self.completed.insert(object_index, outcome).is_some() {82- return Err(format!(83- "Media object {object_index} completed more than once"84- ));85- }86- self.take_next()87- }88-89- fn take_next(&mut self) -> Result<Option<MediaObjectReadOutcome>, String> {90- let Some(outcome) = self.completed.remove(&self.next_object_index) else {91- return Ok(None);92- };93- self.next_object_index = self94- .next_object_index95- .checked_add(1)96- .ok_or_else(|| "Media object reorder index overflowed".to_string())?;97- Ok(Some(outcome))98- }99-}100-101-pub(crate) struct MediaObjectReceiver {102- connection: iroh::endpoint::Connection,103- reads: tokio::task::JoinSet<MediaObjectReadOutcome>,104- reorder: MediaObjectReorder,105- next_object_index: u64,106- connection_closed: bool,107-}108-109-impl MediaObjectReceiver {110- pub(crate) fn new(connection: iroh::endpoint::Connection) -> Self {111- Self {112- connection,113- reads: tokio::task::JoinSet::new(),114- reorder: MediaObjectReorder::new(1),115- next_object_index: 0,116- connection_closed: false,117- }118- }119-120- pub(crate) async fn next(&mut self) -> Result<Option<MediaObjectReadOutcome>, String> {121- loop {122- if let Some(completed) = self.reorder.take_next()? {123- return Ok(Some(completed));124- }125- if self.connection_closed && self.reads.is_empty() {126- if self.reorder.pending_len() != 0 {127- return Err("Media connection closed with an incomplete object order".into());128- }129- return Ok(None);130- }131-132- tokio::select! {133- biased;134- completed = self.reads.join_next(), if !self.reads.is_empty() => {135- let completed = completed136- .ok_or_else(|| "Media object reader ended unexpectedly".to_string())?137- .map_err(|error| format!("Media object reader task failed: {error}"))?;138- if let Some(completed) = self.reorder.push(completed)? {139- return Ok(Some(completed));140- }141- }142- accepted = self.connection.accept_uni(), if !self.connection_closed143- && self.reads.len() + self.reorder.pending_len()144- < CLIENT_MEDIA_OBJECT_CAPACITY => {145- let mut stream = match accepted {146- Ok(stream) => stream,147- Err(_) => {148- self.connection_closed = true;149- continue;150- }151- };152- self.next_object_index = self.next_object_index.checked_add(1)153- .ok_or_else(|| "Media object index overflowed".to_string())?;154- let object_index = self.next_object_index;155- self.reads.spawn(async move {156- match tokio::time::timeout(157- CLIENT_MEDIA_OBJECT_READ_TIMEOUT,158- read_media_object(&mut stream),159- )160- .await161- {162- Err(_) => MediaObjectReadOutcome::Dropped { object_index },163- Ok(Err(ProtocolError::Io(_))) => {164- MediaObjectReadOutcome::Dropped { object_index }165- }166- Ok(Err(error)) => {167- MediaObjectReadOutcome::Malformed(format!(168- "Invalid media object: {error}"169- ))170- }171- Ok(Ok(frame)) => MediaObjectReadOutcome::Frame {172- object_index,173- frame,174- },175- }176- });177- }178- }179- }180- }181-}182-183-impl Drop for MediaObjectReceiver {184- fn drop(&mut self) {185- self.reads.abort_all();186- }187-}188-189-#[derive(Clone, Copy, Debug, PartialEq, Eq)]190-pub(crate) enum MediaObjectSequenceDecision {191- Deliver { discontinuity: bool },192- DropLate,193- DropUntilKeyframe,194-}195-196-#[derive(Debug, Default)]197-pub(crate) struct MediaObjectSequence {198- last_sequence: Option<u64>,199- last_object_index: u64,200- waiting_for_keyframe: bool,201-}202-203-impl MediaObjectSequence {204- pub(crate) fn new() -> Self {205- Self {206- waiting_for_keyframe: true,207- ..Self::default()208- }209- }210-211- pub(crate) fn note_dropped_object(&mut self, object_index: u64) -> bool {212- if object_index <= self.last_object_index {213- return false;214- }215- self.waiting_for_keyframe = true;216- true217- }218-219- pub(crate) fn classify(220- &mut self,221- object_index: u64,222- frame: &MediaFrame,223- ) -> MediaObjectSequenceDecision {224- if object_index <= self.last_object_index225- || self226- .last_sequence227- .is_some_and(|sequence| frame.header.sequence <= sequence)228- {229- return MediaObjectSequenceDecision::DropLate;230- }231-232- let keyframe = frame.header.flags.contains(FrameFlags::KEYFRAME)233- && frame.header.flags.contains(FrameFlags::CODEC_CONFIG);234- let sequence_contiguous = self235- .last_sequence236- .is_none_or(|sequence| sequence.checked_add(1) == Some(frame.header.sequence));237- if !keyframe && (self.waiting_for_keyframe || !sequence_contiguous) {238- self.waiting_for_keyframe = true;239- return MediaObjectSequenceDecision::DropUntilKeyframe;240- }241-242- let discontinuity = frame.header.flags.contains(FrameFlags::DISCONTINUITY)243- || self.waiting_for_keyframe244- || !sequence_contiguous;245- self.last_sequence = Some(frame.header.sequence);246- self.last_object_index = object_index;247- self.waiting_for_keyframe = false;248- MediaObjectSequenceDecision::Deliver { discontinuity }249- }250-}251-25214 #[derive(Debug)]25315 pub(crate) enum MediaObjectReadOutcomeV3 {25416 Object {
@@ -687,168 +449,4 @@ mod tests {
687449 MediaObjectSequenceDecisionV3::DropUntilKeyframe688450 );689451 }690-691- fn media_object_outcome(692- object_index: u64,693- sequence: u64,694- flags: FrameFlags,695- ) -> MediaObjectReadOutcome {696- MediaObjectReadOutcome::Frame {697- object_index,698- frame: media_object_frame(sequence, flags),699- }700- }701-702- fn completed_object_index(outcome: &MediaObjectReadOutcome) -> Option<u64> {703- outcome.object_index()704- }705-706- #[test]707- fn media_object_reorder_restores_accept_order_without_false_resync() {708- let keyframe = FrameFlags::KEYFRAME.union(FrameFlags::CODEC_CONFIG);709- let mut reorder = MediaObjectReorder::new(1);710-711- assert!(712- reorder713- .push(media_object_outcome(2, 11, FrameFlags::NONE))714- .unwrap()715- .is_none()716- );717- assert_eq!(reorder.pending_len(), 1);718- let first = reorder719- .push(media_object_outcome(1, 10, keyframe))720- .unwrap()721- .unwrap();722- assert_eq!(completed_object_index(&first), Some(1));723- assert_eq!(724- completed_object_index(&reorder.take_next().unwrap().unwrap()),725- Some(2)726- );727- assert_eq!(reorder.pending_len(), 0);728- }729-730- #[test]731- fn explicit_discontinuity_keyframe_fast_forwards_bounded_reorder() {732- let barrier = FrameFlags::KEYFRAME733- .union(FrameFlags::CODEC_CONFIG)734- .union(FrameFlags::DISCONTINUITY);735- let mut reorder = MediaObjectReorder::new(1);736-737- assert!(738- reorder739- .push(media_object_outcome(2, 11, FrameFlags::NONE))740- .unwrap()741- .is_none()742- );743- let recovered = reorder744- .push(media_object_outcome(3, 20, barrier))745- .unwrap()746- .unwrap();747- assert_eq!(completed_object_index(&recovered), Some(3));748- assert_eq!(reorder.pending_len(), 0);749- assert_eq!(750- completed_object_index(751- &reorder752- .push(media_object_outcome(1, 10, FrameFlags::NONE))753- .unwrap()754- .unwrap()755- ),756- Some(1)757- );758- }759-760- #[test]761- fn malformed_media_object_bypasses_reorder_and_remains_terminal() {762- let mut reorder = MediaObjectReorder::new(1);763- assert!(764- reorder765- .push(media_object_outcome(2, 11, FrameFlags::NONE))766- .unwrap()767- .is_none()768- );769- assert!(matches!(770- reorder771- .push(MediaObjectReadOutcome::Malformed("bad object".into()))772- .unwrap(),773- Some(MediaObjectReadOutcome::Malformed(_))774- ));775- assert_eq!(reorder.pending_len(), 1);776- }777-778- #[test]779- fn media_objects_begin_on_a_configured_keyframe_then_deliver_contiguously() {780- let keyframe_flags = FrameFlags::KEYFRAME.union(FrameFlags::CODEC_CONFIG);781- let mut sequence = MediaObjectSequence::new();782-783- assert_eq!(784- sequence.classify(1, &media_object_frame(1, FrameFlags::NONE)),785- MediaObjectSequenceDecision::DropUntilKeyframe786- );787- assert_eq!(788- sequence.classify(2, &media_object_frame(2, keyframe_flags)),789- MediaObjectSequenceDecision::Deliver {790- discontinuity: true791- }792- );793- assert_eq!(794- sequence.classify(3, &media_object_frame(3, FrameFlags::NONE)),795- MediaObjectSequenceDecision::Deliver {796- discontinuity: false797- }798- );799- }800-801- #[test]802- fn media_object_sequence_gaps_drop_deltas_until_a_new_keyframe() {803- let keyframe_flags = FrameFlags::KEYFRAME.union(FrameFlags::CODEC_CONFIG);804- let mut sequence = MediaObjectSequence::new();805-806- assert!(matches!(807- sequence.classify(1, &media_object_frame(10, keyframe_flags)),808- MediaObjectSequenceDecision::Deliver { .. }809- ));810- assert_eq!(811- sequence.classify(3, &media_object_frame(12, FrameFlags::NONE)),812- MediaObjectSequenceDecision::DropUntilKeyframe813- );814- assert_eq!(815- sequence.classify(4, &media_object_frame(13, FrameFlags::NONE)),816- MediaObjectSequenceDecision::DropUntilKeyframe817- );818- assert_eq!(819- sequence.classify(5, &media_object_frame(14, keyframe_flags)),820- MediaObjectSequenceDecision::Deliver {821- discontinuity: true822- }823- );824- }825-826- #[test]827- fn late_media_object_completion_cannot_rewind_a_recovered_stream() {828- let keyframe_flags = FrameFlags::KEYFRAME.union(FrameFlags::CODEC_CONFIG);829- let mut sequence = MediaObjectSequence::new();830-831- assert!(matches!(832- sequence.classify(10, &media_object_frame(10, keyframe_flags)),833- MediaObjectSequenceDecision::Deliver { .. }834- ));835- assert!(sequence.note_dropped_object(12));836- assert_eq!(837- sequence.classify(13, &media_object_frame(13, keyframe_flags)),838- MediaObjectSequenceDecision::Deliver {839- discontinuity: true840- }841- );842- assert!(!sequence.note_dropped_object(11));843- assert_eq!(844- sequence.classify(12, &media_object_frame(12, FrameFlags::NONE)),845- MediaObjectSequenceDecision::DropLate846- );847- assert_eq!(848- sequence.classify(14, &media_object_frame(14, FrameFlags::NONE)),849- MediaObjectSequenceDecision::Deliver {850- discontinuity: false851- }852- );853- }854452 }
M src-tauri/src/media/transport.rs +28 -143
@@ -1,20 +1,12 @@
11 use std::time::Duration;22 33 use iroh::Endpoint;4-#[cfg(test)]5-use sigil_protocol::decode_media_frame_object;64 use sigil_protocol::{7- CONTROL_ALPN_V1, Capability, ClientHello, MEDIA_ALPN_V1, MEDIA_ALPN_V2, MEDIA_ALPN_V3,8- PointerSurfaceDimensions, read_host_hello, write_client_hello,5+ CONTROL_ALPN_V1, Capability, ClientHello, MEDIA_ALPN_V3, PointerSurfaceDimensions,6+ read_host_hello, write_client_hello,97 };108 11-pub const MEDIA_TRANSPORT_NAMES: [&str; 5] = [12- "iroh-moq",13- "reliable-v0",14- "reliable-v1",15- "independent-v2",16- "grouped-v3",17-];9+pub const MEDIA_TRANSPORT_NAMES: [&str; 2] = ["iroh-moq", "grouped-v3"];1810 1911 pub(crate) const CLIENT_ENDPOINT_CLOSE_TIMEOUT: Duration = Duration::from_secs(2);2012
@@ -27,9 +19,6 @@ pub(crate) struct NegotiatedV1Stream {
2719 #[derive(Clone, Copy, Debug, PartialEq, Eq)]2820 pub(crate) enum MediaTransport {2921 UpstreamMoq,30- LegacyV0,31- ReliableStreamV1,32- IndependentObjectsV2,3322 GroupedObjectsV3,3423 }3524
@@ -37,16 +26,9 @@ impl MediaTransport {
3726 pub(crate) const fn diagnostic_name(self) -> &'static str {3827 match self {3928 Self::UpstreamMoq => MEDIA_TRANSPORT_NAMES[0],40- Self::LegacyV0 => MEDIA_TRANSPORT_NAMES[1],41- Self::ReliableStreamV1 => MEDIA_TRANSPORT_NAMES[2],42- Self::IndependentObjectsV2 => MEDIA_TRANSPORT_NAMES[3],43- Self::GroupedObjectsV3 => MEDIA_TRANSPORT_NAMES[4],29+ Self::GroupedObjectsV3 => MEDIA_TRANSPORT_NAMES[1],4430 }4531 }46-47- pub(crate) const fn supports_adaptive_feedback(self) -> bool {48- matches!(self, Self::UpstreamMoq | Self::GroupedObjectsV3)49- }5032 }5133 5234 pub(crate) async fn negotiate_v1(
@@ -123,7 +105,7 @@ pub(crate) fn connect_error_is_unsupported_alpn(error: &iroh::endpoint::ConnectE
123105 }124106 }125107 126-async fn open_legacy_negotiated_media_stream(108+pub(crate) async fn open_negotiated_media_stream(127109 endpoint: &Endpoint,128110 address: &iroh::EndpointAddr,129111 nonce: [u8; 16],
@@ -132,152 +114,71 @@ async fn open_legacy_negotiated_media_stream(
132114 (133115 iroh::endpoint::Connection,134116 iroh::endpoint::RecvStream,135- Option<iroh::endpoint::SendStream>,117+ iroh::endpoint::SendStream,136118 NegotiatedV1Stream,137119 MediaTransport,138120 ),139121 String,140122 > {141- match endpoint.connect(address.clone(), MEDIA_ALPN_V3).await {123+ match endpoint.connect(address.clone(), CONTROL_ALPN_V1).await {142124 Ok(connection) => {143125 let (mut send, mut recv) = connection144126 .open_bi()145127 .await146- .map_err(|error| format!("Failed to open media v3 handshake: {error}"))?;128+ .map_err(|error| format!("Failed to open control handshake: {error}"))?;147129 let negotiation = negotiate_v1(148130 &mut send,149131 &mut recv,150132 nonce,151133 vec![Capability::VideoH264],152134 Some(Capability::VideoH264),153- "media v3",135+ "control",154136 invitation,155137 )156138 .await?;139+ // CONTROL owns the authenticated host lease. Keep both the140+ // connection and the client->host send leg alive for keyframe141+ // requests while media uses a separate upstream MoQ session.157142 Ok((158143 connection,159144 recv,160- Some(send),145+ send,161146 negotiation,162- MediaTransport::GroupedObjectsV3,147+ MediaTransport::UpstreamMoq,163148 ))164149 }165- Err(v3_error) if connect_error_is_unsupported_alpn(&v3_error) => {166- match endpoint.connect(address.clone(), MEDIA_ALPN_V2).await {167- Ok(connection) => {168- let (mut send, mut recv) = connection169- .open_bi()170- .await171- .map_err(|error| format!("Failed to open media v2 handshake: {error}"))?;172- let negotiation = negotiate_v1(173- &mut send,174- &mut recv,175- nonce,176- vec![Capability::VideoH264],177- Some(Capability::VideoH264),178- "media v2",179- invitation,180- )181- .await?;182- send.finish()183- .map_err(|error| format!("Failed to finish media v2 handshake: {error}"))?;184- Ok((185- connection,186- recv,187- None,188- negotiation,189- MediaTransport::IndependentObjectsV2,190- ))191- }192- Err(v2_error) if connect_error_is_unsupported_alpn(&v2_error) => {193- let connection = endpoint194- .connect(address.clone(), MEDIA_ALPN_V1)195- .await196- .map_err(|v1_error| {197- format!(198- "Failed to connect media v3 ({v3_error}); v2 compatibility connection failed ({v2_error}); v1 compatibility connection also failed ({v1_error})"199- )200- })?;201- let (mut send, mut recv) = connection202- .open_bi()203- .await204- .map_err(|error| format!("Failed to open media v1 stream: {error}"))?;205- let negotiation = negotiate_v1(206- &mut send,207- &mut recv,208- nonce,209- vec![Capability::VideoH264],210- Some(Capability::VideoH264),211- "media v1",212- invitation,150+ Err(control_error) if connect_error_is_unsupported_alpn(&control_error) => {151+ // Grouped-v3 is the sole compatibility tier below upstream MoQ.152+ let connection = endpoint153+ .connect(address.clone(), MEDIA_ALPN_V3)154+ .await155+ .map_err(|v3_error| {156+ format!(157+ "Control connection failed ({control_error}); media v3 compatibility connection also failed ({v3_error})"213158 )214- .await?;215- send.finish()216- .map_err(|error| format!("Failed to finish media v1 handshake: {error}"))?;217- Ok((218- connection,219- recv,220- None,221- negotiation,222- MediaTransport::ReliableStreamV1,223- ))224- }225- Err(v2_error) => Err(format!(226- "Media v2 compatibility connection failed without an explicit unsupported-ALPN signal; refusing an unsafe downgrade to v1: {v2_error}"227- )),228- }229- }230- Err(v3_error) => Err(format!(231- "Media v3 connection failed without an explicit unsupported-ALPN signal; refusing an unsafe compatibility downgrade: {v3_error}"232- )),233- }234-}235-236-pub(crate) async fn open_negotiated_media_stream(237- endpoint: &Endpoint,238- address: &iroh::EndpointAddr,239- nonce: [u8; 16],240- invitation: Option<&str>,241-) -> Result<242- (243- iroh::endpoint::Connection,244- iroh::endpoint::RecvStream,245- Option<iroh::endpoint::SendStream>,246- NegotiatedV1Stream,247- MediaTransport,248- ),249- String,250-> {251- match endpoint.connect(address.clone(), CONTROL_ALPN_V1).await {252- Ok(connection) => {159+ })?;253160 let (mut send, mut recv) = connection254161 .open_bi()255162 .await256- .map_err(|error| format!("Failed to open control handshake: {error}"))?;163+ .map_err(|error| format!("Failed to open media v3 handshake: {error}"))?;257164 let negotiation = negotiate_v1(258165 &mut send,259166 &mut recv,260167 nonce,261168 vec![Capability::VideoH264],262169 Some(Capability::VideoH264),263- "control",170+ "media v3",264171 invitation,265172 )266173 .await?;267- // CONTROL owns the authenticated host lease. Keep both the268- // connection and the client->host send leg alive for keyframe269- // requests while media uses a separate upstream MoQ session.270174 Ok((271175 connection,272176 recv,273- Some(send),177+ send,274178 negotiation,275- MediaTransport::UpstreamMoq,179+ MediaTransport::GroupedObjectsV3,276180 ))277181 }278- Err(control_error) if connect_error_is_unsupported_alpn(&control_error) => {279- open_legacy_negotiated_media_stream(endpoint, address, nonce, invitation).await280- }281182 Err(control_error) => Err(format!(282183 "Control connection failed without an explicit unsupported-ALPN signal; refusing an unsafe media downgrade: {control_error}"283184 )),
@@ -291,14 +192,10 @@ mod tests {
291192 #[test]292193 fn upstream_moq_transport_is_distinct_from_legacy_compatibility() {293194 assert_eq!(MediaTransport::UpstreamMoq.diagnostic_name(), "iroh-moq");294- assert!(MediaTransport::UpstreamMoq.supports_adaptive_feedback());295- assert!(MediaTransport::GroupedObjectsV3.supports_adaptive_feedback());296- assert!(!MediaTransport::IndependentObjectsV2.supports_adaptive_feedback());297195 assert_ne!(298196 MediaTransport::UpstreamMoq,299197 MediaTransport::GroupedObjectsV3300198 );301- assert!(decode_media_frame_object(&[0_u8; 8]).is_err());302199 }303200 304201 #[test]
@@ -321,22 +218,10 @@ mod tests {
321218 322219 #[test]323220 fn media_transport_names_match_diagnostic_mapping() {324- assert_eq!(325- MEDIA_TRANSPORT_NAMES,326- [327- "iroh-moq",328- "reliable-v0",329- "reliable-v1",330- "independent-v2",331- "grouped-v3",332- ]333- );221+ assert_eq!(MEDIA_TRANSPORT_NAMES, ["iroh-moq", "grouped-v3"]);334222 assert_eq!(335223 [336224 MediaTransport::UpstreamMoq.diagnostic_name(),337- MediaTransport::LegacyV0.diagnostic_name(),338- MediaTransport::ReliableStreamV1.diagnostic_name(),339- MediaTransport::IndependentObjectsV2.diagnostic_name(),340225 MediaTransport::GroupedObjectsV3.diagnostic_name(),341226 ],342227 MEDIA_TRANSPORT_NAMES,
M src-tauri/src/media/video_delivery.rs +35 -297
@@ -1,11 +1,8 @@
1-use std::io::Cursor;21 use std::sync::atomic::{AtomicUsize, Ordering};32 use std::sync::{Arc, Mutex as StdMutex};43 use std::time::{Duration, Instant};54 6-use base64::Engine;7-use openh264::{formats::YUVSource, nal_units};8-use sigil_protocol::{FrameFlags, KeyframeRequestReasonV3, MediaCodec, read_media_frame};5+use sigil_protocol::{FrameFlags, KeyframeRequestReasonV3, MediaCodec};96 use tauri::{107 AppHandle, Emitter,118 ipc::{Channel, Response},
@@ -13,8 +10,8 @@ use tauri::{
1310 1411 use crate::commands::state::MediaControlRequestSender;1512 use crate::media::frame_channel::{16- FrameEnvelopeMetadata, FramePayload, byte_to_codec, emit_frame_error, encode_frame_envelope,17- release_frame_channel_slot, try_reserve_frame_channel_slot, validate_legacy_media_header,13+ FrameEnvelopeMetadata, emit_frame_error, encode_frame_envelope, release_frame_channel_slot,14+ try_reserve_frame_channel_slot,1815 };1916 use crate::media::media_control::try_queue_media_keyframe_request;2017 use crate::media::metrics::{ClientMediaMetrics, lock_client_media_metrics};
@@ -25,13 +22,11 @@ use crate::media::network_diagnostics::{
2522 NetworkLeg, NetworkSessionDiagnostics, lock_network_diagnostics,2623 };2724 use crate::media::object_receiver::{28- MediaObjectReadOutcome, MediaObjectReadOutcomeV3, MediaObjectReceiver, MediaObjectReceiverV3,29- MediaObjectSequence, MediaObjectSequenceDecision, MediaObjectSequenceDecisionV3,25+ MediaObjectReadOutcomeV3, MediaObjectReceiverV3, MediaObjectSequenceDecisionV3,3026 MediaObjectSequenceV3,3127 };3228 use crate::media::transport::MediaTransport;3329 34-const LEGACY_MEDIA_IDLE_TIMEOUT: Duration = Duration::from_secs(10);3530 const CLIENT_FRAME_STATS_INTERVAL: Duration = Duration::from_millis(250);3631 3732 pub(crate) struct VideoDeliveryRequest {
@@ -49,14 +44,15 @@ pub(crate) struct VideoDeliveryRequest {
4944 pub(crate) media_transport: MediaTransport,5045 pub(crate) media_generation: u64,5146 pub(crate) connected_audio_generation: Option<u64>,52- pub(crate) use_webcodecs: bool,5347 }5448 5549 pub(crate) async fn run_video_delivery(request: VideoDeliveryRequest) {5650 let VideoDeliveryRequest {5751 app,5852 endpoint,59- mut frame_recv,53+ // The negotiated control/v3 handshake recv leg carries no media but54+ // must stay open for the session's lifetime.55+ frame_recv: _control_recv_keepalive,6056 frame_connection: frame_connection_for_stats,6157 input_connection,6258 audio_connection: audio_connection_for_stats,
@@ -68,7 +64,6 @@ pub(crate) async fn run_video_delivery(request: VideoDeliveryRequest) {
6864 media_transport,6965 media_generation,7066 connected_audio_generation,71- use_webcodecs,7267 } = request;7368 7469 let metrics_started = Instant::now();
@@ -80,34 +75,11 @@ pub(crate) async fn run_video_delivery(request: VideoDeliveryRequest) {
8075 let metrics = Arc::new(StdMutex::new(initial_metrics));8176 let mut previous_sequence: Option<u64> = None;8277 let mut frontend_waiting_for_keyframe = true;83- let mut media_objects = (media_transport == MediaTransport::IndependentObjectsV2)84- .then(|| MediaObjectReceiver::new(frame_connection_for_stats.clone()));85- let mut media_object_sequence = MediaObjectSequence::new();8678 let mut media_objects_v3 = (media_transport == MediaTransport::GroupedObjectsV3)8779 .then(|| MediaObjectReceiverV3::new(frame_connection_for_stats.clone()));8880 let mut media_object_sequence_v3 = MediaObjectSequenceV3::new();8981 let mut upstream_moq_media = upstream_moq_media;9082 91- let mut decoder = if use_webcodecs {92- None93- } else {94- match openh264::decoder::Decoder::new() {95- Ok(d) => Some(d),96- Err(e) => {97- emit_frame_error(&app, media_generation, format!("Decoder init failed: {e}"));98- if media_transport == MediaTransport::UpstreamMoq {99- retire_upstream_moq_generation(100- &app,101- media_generation,102- connected_audio_generation,103- )104- .await;105- }106- return;107- }108- }109- };110-11183 let (stats_stop, mut stats_stop_rx) = tokio::sync::watch::channel(false);11284 let stats_app = app.clone();11385 let stats_metrics = Arc::clone(&metrics);
@@ -308,180 +280,6 @@ pub(crate) async fn run_video_delivery(request: VideoDeliveryRequest) {
308280 }309281 }310282 }311- MediaTransport::IndependentObjectsV2 => {312- let receiver = media_objects313- .as_mut()314- .expect("media v2 receiver must exist for media v2 transport");315- loop {316- let outcome = match receiver.next().await {317- Ok(Some(outcome)) => outcome,318- Ok(None) => {319- emit_frame_error(&app, media_generation, "Connection closed");320- break 'frames;321- }322- Err(error) => {323- emit_frame_error(&app, media_generation, error);324- break 'frames;325- }326- };327- match outcome {328- MediaObjectReadOutcome::Dropped { object_index } => {329- let begins_resync =330- media_object_sequence.note_dropped_object(object_index);331- let mut metrics = lock_client_media_metrics(&metrics);332- metrics.observe_transport_object_drop(!begins_resync);333- if begins_resync {334- frontend_waiting_for_keyframe = true;335- metrics.begin_frontend_resync(metrics_started.elapsed());336- }337- }338- MediaObjectReadOutcome::Malformed(error) => {339- emit_frame_error(&app, media_generation, error);340- break 'frames;341- }342- MediaObjectReadOutcome::Frame {343- object_index,344- frame,345- } => {346- let discontinuity =347- match media_object_sequence.classify(object_index, &frame) {348- MediaObjectSequenceDecision::Deliver { discontinuity } => {349- discontinuity350- }351- MediaObjectSequenceDecision::DropLate => {352- lock_client_media_metrics(&metrics)353- .observe_transport_object_drop(true);354- continue;355- }356- MediaObjectSequenceDecision::DropUntilKeyframe => {357- frontend_waiting_for_keyframe = true;358- let mut metrics = lock_client_media_metrics(&metrics);359- metrics.observe_transport_object_drop(false);360- metrics.begin_frontend_resync(metrics_started.elapsed());361- continue;362- }363- };364- let codec = match frame.header.codec {365- MediaCodec::H264 => "h264".to_string(),366- };367- break (368- u32::from(frame.header.width),369- u32::from(frame.header.height),370- frame.payload,371- frame.header.flags.contains(FrameFlags::KEYFRAME),372- codec,373- Some(frame.header.sequence),374- Some(frame.header.capture_timestamp_us),375- Some(frame.header.pts_us),376- discontinuity,377- frame.header.flags.contains(FrameFlags::CODEC_CONFIG),378- );379- }380- }381- }382- }383- MediaTransport::ReliableStreamV1 => {384- // Gamescope's PipeWire stream is damage-driven: a static385- // screen can legitimately produce no encoded frame for an386- // arbitrary period. Connection closure and parser/source387- // errors remain terminal; frame silence does not.388- let frame = match read_media_frame(&mut frame_recv).await {389- Ok(Some(frame)) => frame,390- Ok(None) => {391- emit_frame_error(&app, media_generation, "Connection closed");392- break;393- }394- Err(error) => {395- emit_frame_error(396- &app,397- media_generation,398- format!("Invalid media stream: {error}"),399- );400- break;401- }402- };403- let codec = match frame.header.codec {404- MediaCodec::H264 => "h264".to_string(),405- };406- (407- u32::from(frame.header.width),408- u32::from(frame.header.height),409- frame.payload,410- frame.header.flags.contains(FrameFlags::KEYFRAME),411- codec,412- Some(frame.header.sequence),413- Some(frame.header.capture_timestamp_us),414- Some(frame.header.pts_us),415- frame.header.flags.contains(FrameFlags::DISCONTINUITY),416- frame.header.flags.contains(FrameFlags::CODEC_CONFIG),417- )418- }419- MediaTransport::LegacyV0 => {420- let mut header = [0u8; 14];421- match tokio::time::timeout(422- LEGACY_MEDIA_IDLE_TIMEOUT,423- frame_recv.read_exact(&mut header),424- )425- .await426- {427- Err(_) => {428- emit_frame_error(&app, media_generation, "Media stream idle timeout");429- break;430- }431- Ok(Err(_)) => {432- emit_frame_error(&app, media_generation, "Connection lost");433- break;434- }435- Ok(Ok(_)) => {}436- }437- let w = u32::from_be_bytes([header[0], header[1], header[2], header[3]]);438- let h = u32::from_be_bytes([header[4], header[5], header[6], header[7]]);439- let frame_len =440- u32::from_be_bytes([header[8], header[9], header[10], header[11]]) as usize;441- if let Err(error) = validate_legacy_media_header(w, h, frame_len) {442- emit_frame_error(&app, media_generation, error);443- break;444- }445-446- let is_keyframe = header[12] == 1;447- let codec = match byte_to_codec(header[13]) {448- Ok(codec) => codec.to_string(),449- Err(error) => {450- emit_frame_error(&app, media_generation, error);451- break;452- }453- };454-455- let mut frame_buf = vec![0u8; frame_len];456- match tokio::time::timeout(457- LEGACY_MEDIA_IDLE_TIMEOUT,458- frame_recv.read_exact(&mut frame_buf),459- )460- .await461- {462- Err(_) => {463- emit_frame_error(&app, media_generation, "Media payload idle timeout");464- break;465- }466- Ok(Err(_)) => {467- emit_frame_error(&app, media_generation, "Connection lost");468- break;469- }470- Ok(Ok(_)) => {}471- }472- (473- w,474- h,475- frame_buf,476- is_keyframe,477- codec,478- None,479- None,480- None,481- false,482- false,483- )484- }485283 };486284 lock_client_media_metrics(&metrics).observe_transport_receive(487285 metrics_started.elapsed(),
@@ -526,100 +324,40 @@ pub(crate) async fn run_video_delivery(request: VideoDeliveryRequest) {
526324 lock_client_media_metrics(&metrics)527325 .observe_frontend_queue_depth(frame_events_in_flight.load(Ordering::SeqCst));528326 529- let delivered_to_frontend;530- if use_webcodecs {531- let envelope = match encode_frame_envelope(532- FrameEnvelopeMetadata {533- width: w,534- height: h,535- codec: &codec,536- keyframe: is_keyframe,537- discontinuity,538- codec_config,539- sequence,540- capture_timestamp_micros,541- pts_micros,542- },543- &frame_buf,544- ) {545- Ok(envelope) => envelope,546- Err(error) => {547- emit_frame_error(&app, media_generation, error);548- release_frame_channel_slot(&frame_events_in_flight);549- break;550- }551- };552- let ipc_send_started = Instant::now();553- let send_result = frame_channel.send(Response::new(envelope));554- lock_client_media_metrics(&metrics).observe_frontend_ipc_send_duration(555- metrics_started.elapsed(),556- ipc_send_started.elapsed(),557- );558- if send_result.is_err() {327+ let envelope = match encode_frame_envelope(328+ FrameEnvelopeMetadata {329+ width: w,330+ height: h,331+ codec: &codec,332+ keyframe: is_keyframe,333+ discontinuity,334+ codec_config,335+ sequence,336+ capture_timestamp_micros,337+ pts_micros,338+ },339+ &frame_buf,340+ ) {341+ Ok(envelope) => envelope,342+ Err(error) => {343+ emit_frame_error(&app, media_generation, error);559344 release_frame_channel_slot(&frame_events_in_flight);560345 break;561346 }562- delivered_to_frontend = true;563- } else if let Some(ref mut dec) = decoder {564- let mut emitted = false;565- for nal in nal_units(&frame_buf) {566- if let Ok(Some(yuv)) = dec.decode(nal) {567- let (yw, yh) = yuv.dimensions();568- let rgb_len = yuv.rgb8_len();569- let mut rgb_raw = vec![0u8; rgb_len];570- yuv.write_rgb8(&mut rgb_raw);571-572- let img = match image::RgbImage::from_raw(yw as u32, yh as u32, rgb_raw) {573- Some(img) => img,574- None => continue,575- };576- let mut jpeg_buf = Vec::with_capacity(30_000);577- if image::DynamicImage::ImageRgb8(img)578- .write_to(&mut Cursor::new(&mut jpeg_buf), image::ImageFormat::Jpeg)579- .is_err()580- {581- continue;582- }583-584- let b64 = base64::engine::general_purpose::STANDARD.encode(&jpeg_buf);585- let ipc_send_started = Instant::now();586- let send_result = app.emit(587- "frame",588- FramePayload {589- generation: media_generation,590- width: yw as u32,591- height: yh as u32,592- data: b64,593- keyframe: is_keyframe,594- codec: codec.clone(),595- capture_timestamp_micros,596- pts_micros,597- discontinuity,598- },599- );600- lock_client_media_metrics(&metrics).observe_frontend_ipc_send_duration(601- metrics_started.elapsed(),602- ipc_send_started.elapsed(),603- );604- if send_result.is_err() {605- release_frame_channel_slot(&frame_events_in_flight);606- break 'frames;607- }608- emitted = true;609- break;610- }611- }612- if !emitted {613- release_frame_channel_slot(&frame_events_in_flight);614- }615- delivered_to_frontend = emitted;616- } else {347+ };348+ let ipc_send_started = Instant::now();349+ let send_result = frame_channel.send(Response::new(envelope));350+ lock_client_media_metrics(&metrics).observe_frontend_ipc_send_duration(351+ metrics_started.elapsed(),352+ ipc_send_started.elapsed(),353+ );354+ if send_result.is_err() {617355 release_frame_channel_slot(&frame_events_in_flight);618- delivered_to_frontend = false;356+ break;619357 }620358 621- if delivered_to_frontend {622- frontend_waiting_for_keyframe = false;359+ frontend_waiting_for_keyframe = false;360+ {623361 let mut metrics = lock_client_media_metrics(&metrics);624362 let elapsed = metrics_started.elapsed();625363 metrics.observe_frontend_send(elapsed);