refactor(portal): extract input delivery

Dami committed Jul 223f5d54c

commit: 3f5d54cf81a046b8c880643b98cfd10c8fa1cce8
parent: 7770f8042e525c7e55cb05bb3c99cef312250642
author: Dami <dami.etoile@gmail.com>

5 files changed, +975 -886wrap lines

M	src-tauri/src/commands/network.rs	930	+-----------------------------
M	src-tauri/src/commands/state.rs	2	-
A	src-tauri/src/media/input_delivery.rs	919	++++++++++++++++++++++++++++++
M	src-tauri/src/media/mod.rs	1	+
M	src-tauri/src/media/network_diagnostics.rs	9	+

M src-tauri/src/commands/network.rs +46 -884

@@ -1,9 +1,6 @@
11 use super::auth::derive_iroh_secret_from_key;22 use super::enrollment::{connection_enrollment, mark_invitation_redeemed};3-use super::state::{4-    AppState, CLIENT_INPUT_QUEUE_CAPACITY, FRAME_ALPN, INPUT_ALPN, MediaFeedbackSender,5-    development_direct_node_available,6-};3+use super::state::{AppState, FRAME_ALPN, MediaFeedbackSender, development_direct_node_available};74 pub use crate::media::adaptive_feedback::ClientMediaFeedbackReport;85 use crate::media::adaptive_feedback::{96     CLIENT_MEDIA_FEEDBACK_CONNECT_TIMEOUT, open_negotiated_feedback_stream,@@ -19,36 +16,41 @@ use crate::media::frame_channel::{
1916     release_frame_channel_slot_for_generation, try_reserve_frame_channel_slot,2017     validate_legacy_media_header,2118 };19+use crate::media::input_delivery::{20+    CLIENT_INPUT_QUEUE_CAPACITY, InputSession, open_input_session, run_input_feedback,21+    run_input_forwarder, send_input,22+};23+#[allow(unused_imports)]24+pub use crate::media::input_delivery::{PointerFeedbackPayload, PointerFeedbackTerminalReason};2225 use crate::media::metrics::{ClientMediaMetrics, lock_client_media_metrics};2326 use crate::media::moq_receiver::{2427     MoqMediaReadOutcome, open_upstream_moq_media, retire_upstream_moq_generation,2528 };26-use crate::media::network_diagnostics::{NetworkLeg, NetworkSessionDiagnostics};29+use crate::media::network_diagnostics::{30+    NetworkLeg, NetworkSessionDiagnostics, lock_network_diagnostics,31+};2732 use crate::media::object_receiver::{2833     CLIENT_MEDIA_OBJECT_CAPACITY, CLIENT_MEDIA_OBJECT_READ_TIMEOUT, MediaObjectReadOutcome,2934     MediaObjectReceiver, MediaObjectSequence, MediaObjectSequenceDecision,3035 };3136 use crate::media::transport::{32-    CLIENT_ENDPOINT_CLOSE_TIMEOUT, MediaTransport, NegotiatedV1Stream, negotiate_v1,33-    open_negotiated_media_stream,37+    CLIENT_ENDPOINT_CLOSE_TIMEOUT, MediaTransport, open_negotiated_media_stream,3438 };3539 use base64::Engine;3640 use iroh::{Endpoint, SecretKey, endpoint::presets};3741 use openh264::{formats::YUVSource, nal_units};3842 use serde::Serialize;3943 use sigil_protocol::{40-    Capability, FrameFlags, INPUT_ALPN_V1, InputEvent, InvitationGrants, KeyframeRequestReasonV3,41-    MAX_MEDIA_GROUP_BYTES_V3, MediaCodec, MediaControlRequestV3, MediaObjectV3, PointerPosition,42-    PointerSurfaceDimensions, ProtocolError, RELATIVE_POINTER_DELTA_MAX,43-    RELATIVE_POINTER_DELTA_MIN, read_input_ack, read_media_frame, read_media_object_v3,44-    write_input_event, write_media_control_request_v3,44+    FrameFlags, InputEvent, InvitationGrants, KeyframeRequestReasonV3, MAX_MEDIA_GROUP_BYTES_V3,45+    MediaCodec, MediaControlRequestV3, MediaObjectV3, PointerSurfaceDimensions, ProtocolError,46+    read_media_frame, read_media_object_v3, write_media_control_request_v3,4547 };4648 use std::collections::BTreeMap;4749 use std::io::Cursor;4850 #[cfg(test)]4951 use std::sync::atomic::AtomicU64;5052 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};51-use std::sync::{Arc, Mutex as StdMutex, MutexGuard as StdMutexGuard};53+use std::sync::{Arc, Mutex as StdMutex};5254 use std::time::{Duration, Instant};5355 use tauri::{5456     AppHandle, Emitter, State,@@ -60,14 +62,6 @@ use tauri::{
6062 const LEGACY_MEDIA_IDLE_TIMEOUT: Duration = Duration::from_secs(10);6163 const CLIENT_FRAME_STATS_INTERVAL: Duration = Duration::from_millis(250);6264 63-fn lock_network_diagnostics(64-    diagnostics: &StdMutex<NetworkSessionDiagnostics>,65-) -> StdMutexGuard<'_, NetworkSessionDiagnostics> {66-    diagnostics67-        .lock()68-        .unwrap_or_else(std::sync::PoisonError::into_inner)69-}70-7165 struct ClientConnectGuard {7266     active: Arc<AtomicBool>,7367     committed: bool,@@ -390,144 +384,6 @@ impl MediaObjectSequenceV3 {
390384     }391385 }392386 393-#[derive(Debug, Clone, Copy, PartialEq, Eq)]394-struct InputAvailability {395-    relative_pointer: bool,396-    pointer_position_feedback: bool,397-    absolute_pointer: bool,398-    keyboard: bool,399-    text: bool,400-    gamepad: bool,401-    input_ack: bool,402-    control: bool,403-}404-405-impl InputAvailability {406-    fn from_capabilities(capabilities: &[Capability]) -> Self {407-        let relative_pointer = capabilities.contains(&Capability::RelativePointer);408-        let pointer_position_feedback = capabilities.contains(&Capability::PointerPositionFeedback);409-        let absolute_pointer = capabilities.contains(&Capability::AbsolutePointer);410-        let keyboard = capabilities.contains(&Capability::Keyboard);411-        let text = capabilities.contains(&Capability::Text);412-        let gamepad = capabilities.contains(&Capability::Gamepad);413-        let input_ack = capabilities.contains(&Capability::InputAck);414-        Self {415-            relative_pointer,416-            pointer_position_feedback,417-            absolute_pointer,418-            keyboard,419-            text,420-            gamepad,421-            input_ack,422-            control: relative_pointer || absolute_pointer || keyboard || text || gamepad,423-        }424-    }425-}426-427-#[derive(Serialize, Clone, Copy)]428-#[serde(tag = "type", rename_all = "snake_case")]429-pub enum PointerFeedbackPayload {430-    Position {431-        sequence: u64,432-        position: Option<PointerPosition>,433-        #[serde(skip_serializing_if = "Option::is_none")]434-        pointer_visible: Option<bool>,435-    },436-    Terminal {437-        reason: PointerFeedbackTerminalReason,438-    },439-}440-441-#[derive(Serialize, Clone, Copy)]442-#[serde(rename_all = "snake_case")]443-pub enum PointerFeedbackTerminalReason {444-    Eof,445-    Malformed,446-}447-448-#[derive(Debug, Default, PartialEq, Eq)]449-struct RelativePointerAccumulator {450-    dx: i64,451-    dy: i64,452-}453-454-impl RelativePointerAccumulator {455-    fn push(&mut self, dx: i32, dy: i32) {456-        self.dx = self.dx.saturating_add(i64::from(dx));457-        self.dy = self.dy.saturating_add(i64::from(dy));458-    }459-460-    fn take(&mut self) -> Option<InputEvent> {461-        if self.dx == 0 && self.dy == 0 {462-            return None;463-        }464-        let dx = self.dx.clamp(465-            i64::from(RELATIVE_POINTER_DELTA_MIN),466-            i64::from(RELATIVE_POINTER_DELTA_MAX),467-        ) as i32;468-        let dy = self.dy.clamp(469-            i64::from(RELATIVE_POINTER_DELTA_MIN),470-            i64::from(RELATIVE_POINTER_DELTA_MAX),471-        ) as i32;472-        let event = InputEvent::MouseMoveRelative { dx, dy };473-        self.dx -= i64::from(dx);474-        self.dy -= i64::from(dy);475-        Some(event)476-    }477-478-    fn is_pending(&self) -> bool {479-        self.dx != 0 || self.dy != 0480-    }481-}482-483-fn stage_relative_input(484-    pending: &mut RelativePointerAccumulator,485-    event: InputEvent,486-) -> Option<InputEvent> {487-    match event {488-        InputEvent::MouseMoveRelative { dx, dy } => {489-            pending.push(dx, dy);490-            None491-        }492-        event => Some(event),493-    }494-}495-496-async fn open_negotiated_input_stream(497-    endpoint: &Endpoint,498-    address: &iroh::EndpointAddr,499-    nonce: [u8; 16],500-    capabilities: Vec<Capability>,501-) -> Result<502-    (503-        iroh::endpoint::Connection,504-        iroh::endpoint::SendStream,505-        iroh::endpoint::RecvStream,506-        NegotiatedV1Stream,507-    ),508-    String,509-> {510-    let connection = endpoint511-        .connect(address.clone(), INPUT_ALPN_V1)512-        .await513-        .map_err(|error| format!("Failed to connect input stream: {error}"))?;514-    let (mut send, mut recv) = connection515-        .open_bi()516-        .await517-        .map_err(|error| format!("Failed to open input stream: {error}"))?;518-    let negotiation = negotiate_v1(519-        &mut send,520-        &mut recv,521-        nonce,522-        capabilities,523-        None,524-        "input",525-        None,526-    )527-    .await?;528-    Ok((connection, send, recv, negotiation))529-}530-531387 async fn run_media_control_writer_v3(532388     mut stream: iroh::endpoint::SendStream,533389     mut requests: tokio::sync::mpsc::Receiver<(KeyframeRequestReasonV3, Option<u64>)>,@@ -583,123 +439,6 @@ fn try_queue_media_keyframe_request(
583439     }584440 }585441 586-fn input_capability_offers(grants: InvitationGrants) -> Vec<Vec<Capability>> {587-    let base = [588-        vec![589-            Capability::RelativePointer,590-            Capability::PointerPositionFeedback,591-            Capability::PointerVisibilityFeedback,592-            Capability::AbsolutePointer,593-            Capability::Keyboard,594-            Capability::Text,595-            Capability::Gamepad,596-        ],597-        vec![598-            Capability::RelativePointer,599-            Capability::PointerPositionFeedback,600-            Capability::AbsolutePointer,601-            Capability::Keyboard,602-            Capability::Text,603-            Capability::Gamepad,604-        ],605-        vec![606-            Capability::RelativePointer,607-            Capability::AbsolutePointer,608-            Capability::Keyboard,609-            Capability::Text,610-            Capability::Gamepad,611-        ],612-        vec![613-            Capability::AbsolutePointer,614-            Capability::Keyboard,615-            Capability::Text,616-            Capability::Gamepad,617-        ],618-    ];619-    let has_input_grant = grants.contains(InvitationGrants::POINTER_KEYBOARD)620-        || grants.contains(InvitationGrants::GAMEPAD);621-    let mut offers = Vec::with_capacity(base.len() * 2);622-    if has_input_grant {623-        for mut offer in base.clone() {624-            offer.push(Capability::InputAck);625-            offers.push(offer);626-        }627-    }628-    offers.extend(base);629-    for offer in &mut offers {630-        offer.retain(|capability| match capability {631-            Capability::Gamepad => grants.contains(InvitationGrants::GAMEPAD),632-            Capability::AbsolutePointer633-            | Capability::RelativePointer634-            | Capability::Keyboard635-            | Capability::Text636-            | Capability::PointerPositionFeedback637-            | Capability::PointerVisibilityFeedback => {638-                grants.contains(InvitationGrants::POINTER_KEYBOARD)639-            }640-            Capability::InputAck => has_input_grant,641-            _ => true,642-        });643-    }644-    offers.dedup();645-    offers646-}647-648-fn input_event_allowed(capabilities: &[Capability], event: &InputEvent) -> bool {649-    match event {650-        InputEvent::Probe => capabilities.contains(&Capability::InputAck),651-        InputEvent::MouseMove { .. } => capabilities.contains(&Capability::AbsolutePointer),652-        InputEvent::MouseMoveRelative { .. } => capabilities.contains(&Capability::RelativePointer),653-        InputEvent::MousePositionSync { .. } => capabilities.contains(&Capability::RelativePointer),654-        InputEvent::MouseClick { .. }655-        | InputEvent::MouseDown { .. }656-        | InputEvent::MouseUp { .. }657-        | InputEvent::MouseScroll { .. } => {658-            capabilities.contains(&Capability::RelativePointer)659-                || capabilities.contains(&Capability::AbsolutePointer)660-        }661-        InputEvent::KeyDown { .. } | InputEvent::KeyUp { .. } | InputEvent::KeyClick { .. } => {662-            capabilities.contains(&Capability::Keyboard)663-        }664-        InputEvent::Text { .. } => capabilities.contains(&Capability::Text),665-        InputEvent::Gamepad { .. } => capabilities.contains(&Capability::Gamepad),666-    }667-}668-669-async fn write_client_input_event(670-    stream: &mut iroh::endpoint::SendStream,671-    event: &InputEvent,672-    use_v1: bool,673-    diagnostics: Option<&Arc<StdMutex<NetworkSessionDiagnostics>>>,674-) -> Result<(), String> {675-    if let Some(diagnostics) = diagnostics {676-        lock_network_diagnostics(diagnostics).begin_input_send(Instant::now());677-    }678-    if use_v1 {679-        write_input_event(stream, event)680-            .await681-            .map_err(|error| error.to_string())682-    } else {683-        let json = serde_json::to_string(event).map_err(|error| error.to_string())?;684-        stream685-            .write_all(format!("{json}\n").as_bytes())686-            .await687-            .map_err(|error| error.to_string())688-    }689-}690-691-fn observe_input_ack_if_negotiated(692-    diagnostics: &StdMutex<NetworkSessionDiagnostics>,693-    negotiated: bool,694-    sequence: u64,695-    now: Instant,696-) -> Result<(), String> {697-    if !negotiated {698-        return Ok(());699-    }700-    lock_network_diagnostics(diagnostics).observe_input_ack(sequence, now)701-}702-703442 #[tauri::command]704443 pub async fn iroh_client_connect(705444     app: AppHandle,@@ -781,8 +520,6 @@ pub async fn iroh_client_connect(
781520     // protocols. The inherited v0 leg is retained below solely as migration782521     // code and is no longer selected by an ordinary Portal connection.783522     let use_v1 = true;784-    let input_alpn = if use_v1 { INPUT_ALPN_V1 } else { INPUT_ALPN };785-786523     let (frame_conn, mut frame_recv, media_control_stream, media_negotiation, media_transport) =787524         if use_v1 {788525             let first_attempt = open_negotiated_media_stream(@@ -899,115 +636,38 @@ pub async fn iroh_client_connect(
899636         (None, None)900637     };901638 902-    let (input_connection, input_send, input_recv, input_capabilities) = if use_v1 {903-        let mut errors = Vec::new();904-        let mut accepted = None;905-        // Older hosts reject unknown capability enum values. Try all four906-        // pointer feature levels with ACK first, then repeat the exact legacy907-        // offers without ACK so lack of diagnostics never forces absolute908-        // pointer input prematurely.909-        for capabilities in input_capability_offers(grants) {910-            match open_negotiated_input_stream(&endpoint, &addr, handshake_nonce, capabilities)911-                .await912-            {913-                Ok(result) => {914-                    accepted = Some(result);915-                    break;916-                }917-                Err(error) => errors.push(error),918-            }919-        }920-        let (connection, send, recv, input_negotiation) = accepted921-            .ok_or_else(|| format!("All input capability offers failed: {}", errors.join("; ")))?;922-        if Some(input_negotiation.session_id) != media_session_id {923-            return Err("Host returned mismatched media and input sessions".to_string());924-        }925-        (connection, send, recv, input_negotiation.capabilities)926-    } else {927-        let input_conn = endpoint928-            .connect(addr.clone(), input_alpn)929-            .await930-            .map_err(|e| format!("Failed to connect input stream: {}", e))?;931-        let (mut send, recv) = input_conn932-            .open_bi()933-            .await934-            .map_err(|e| format!("Failed to open input stream: {}", e))?;935-        send.write_all(&[1u8])936-            .await937-            .map_err(|e| format!("Failed to send input start: {}", e))?;938-        // The inherited protocol predates negotiation and supports the current939-        // absolute-pointer, keyboard, and text event set.940-        (941-            input_conn,942-            send,943-            recv,944-            vec![945-                Capability::AbsolutePointer,946-                Capability::Keyboard,947-                Capability::Text,948-            ],949-        )950-    };639+    let InputSession {640+        connection: input_connection,641+        send: input_send,642+        recv: input_recv,643+        capabilities: input_capabilities,644+        availability: input_availability,645+    } = open_input_session(646+        &endpoint,647+        &addr,648+        handshake_nonce,649+        media_session_id,650+        grants,651+        use_v1,652+    )653+    .await?;951654 952-    let input_availability = InputAvailability::from_capabilities(&input_capabilities);953655     let network_diagnostics = Arc::new(StdMutex::new(NetworkSessionDiagnostics::new(954656         Instant::now(),955657         input_availability.input_ack,956658     )));957659 958660     if input_availability.pointer_position_feedback || input_availability.input_ack {959-        let mut input_feedback = input_recv;960661         let feedback_diagnostics = Arc::clone(&network_diagnostics);961-        let mut pointer_feedback_enabled = input_availability.pointer_position_feedback;662+        let pointer_feedback_enabled = input_availability.pointer_position_feedback;962663         let input_ack_enabled = input_availability.input_ack;963-        tokio::spawn(async move {964-            let terminal_reason = loop {965-                let response = match read_input_ack(&mut input_feedback).await {966-                    Ok(Some(response)) => response,967-                    Ok(None) => {968-                        lock_network_diagnostics(&feedback_diagnostics)969-                            .mark_input_feedback_closed();970-                        break PointerFeedbackTerminalReason::Eof;971-                    }972-                    Err(error) => {973-                        lock_network_diagnostics(&feedback_diagnostics)974-                            .mark_input_feedback_malformed();975-                        eprintln!("[client] invalid input feedback: {error}");976-                        break PointerFeedbackTerminalReason::Malformed;977-                    }978-                };979-                if let Err(error) = observe_input_ack_if_negotiated(980-                    &feedback_diagnostics,981-                    input_ack_enabled,982-                    response.sequence,983-                    Instant::now(),984-                ) {985-                    eprintln!("[client] invalid input acknowledgement: {error}");986-                    break PointerFeedbackTerminalReason::Malformed;987-                }988-                if pointer_feedback_enabled989-                    && pointer_channel990-                        .send(PointerFeedbackPayload::Position {991-                            sequence: response.sequence,992-                            position: response.pointer_position,993-                            pointer_visible: response.pointer_visible,994-                        })995-                        .is_err()996-                {997-                    // Losing the webview's pointer channel must not stop ACK998-                    // draining and apply backpressure to host input.999-                    pointer_feedback_enabled = false;1000-                    if !input_ack_enabled {1001-                        return;1002-                    }1003-                }1004-            };1005-            // The session-owned channel emits at most one terminal message.1006-            // JavaScript rejects deliveries from superseded channel closures.1007-            let _ = pointer_channel.send(PointerFeedbackPayload::Terminal {1008-                reason: terminal_reason,1009-            });1010-        });664+        tokio::spawn(run_input_feedback(665+            input_recv,666+            feedback_diagnostics,667+            pointer_feedback_enabled,668+            input_ack_enabled,669+            pointer_channel,670+        ));1011671     } else {1012672         drop(input_recv);1013673     }@@ -1089,130 +749,14 @@ pub async fn iroh_client_connect(
1089749     // Input forwarder: absolute motion is latest-value state and may be1090750     // dropped at the 60 Hz boundary. Relative motion is displacement, so it1091751     // owns a separate accumulator and timer that coalesces rather than drops.1092-    let mut input_stream = input_send;1093752     let input_send_diagnostics = Arc::clone(&network_diagnostics);1094-    tokio::spawn(async move {1095-        let mut rx = rx;1096-        const MOUSE_INTERVAL: Duration = Duration::from_millis(16);1097-        let started = Instant::now();1098-        let mut last_absolute_mouse_time = started.checked_sub(MOUSE_INTERVAL).unwrap_or(started);1099-        let mut last_relative_mouse_time = started.checked_sub(MOUSE_INTERVAL).unwrap_or(started);1100-        let mut pending_relative = RelativePointerAccumulator::default();1101-        let mut input_open = true;1102-1103-        while input_open {1104-            let event = if pending_relative.is_pending() {1105-                let wait = MOUSE_INTERVAL.saturating_sub(last_relative_mouse_time.elapsed());1106-                if wait.is_zero() {1107-                    let Some(event) = pending_relative.take() else {1108-                        continue;1109-                    };1110-                    if let Err(error) = write_client_input_event(1111-                        &mut input_stream,1112-                        &event,1113-                        use_v1,1114-                        Some(&input_send_diagnostics),1115-                    )1116-                    .await1117-                    {1118-                        eprintln!("[client] input stream write failed: {error}; disconnecting");1119-                        break;1120-                    }1121-                    last_relative_mouse_time = Instant::now();1122-                    continue;1123-                }1124-                tokio::select! {1125-                    event = rx.recv() => event,1126-                    () = tokio::time::sleep(wait) => {1127-                        let Some(event) = pending_relative.take() else {1128-                            continue;1129-                        };1130-                        if let Err(error) = write_client_input_event(1131-                            &mut input_stream,1132-                            &event,1133-                            use_v1,1134-                            Some(&input_send_diagnostics),1135-                        ).await {1136-                            eprintln!("[client] input stream write failed: {error}; disconnecting");1137-                            break;1138-                        }1139-                        last_relative_mouse_time = Instant::now();1140-                        continue;1141-                    }1142-                }1143-            } else {1144-                rx.recv().await1145-            };1146-            let Some(event) = event else {1147-                input_open = false;1148-                continue;1149-            };1150-            // The host's accepted capability set is an authorization boundary.1151-            // Drop unavailable event classes silently so event contents never1152-            // reach logs or the wire even if a compromised webview invokes the1153-            // command directly.1154-            if !input_event_allowed(&input_capabilities, &event) {1155-                continue;1156-            }1157-            let Some(event) = stage_relative_input(&mut pending_relative, event) else {1158-                continue;1159-            };1160-            let mut flushed_relative_barrier = false;1161-            while let Some(relative_barrier) = pending_relative.take() {1162-                if let Err(error) = write_client_input_event(1163-                    &mut input_stream,1164-                    &relative_barrier,1165-                    use_v1,1166-                    Some(&input_send_diagnostics),1167-                )1168-                .await1169-                {1170-                    eprintln!("[client] input stream write failed: {error}; disconnecting");1171-                    input_open = false;1172-                    break;1173-                }1174-                flushed_relative_barrier = true;1175-            }1176-            if !input_open {1177-                break;1178-            }1179-            if flushed_relative_barrier {1180-                last_relative_mouse_time = Instant::now();1181-            }1182-            if matches!(event, InputEvent::MouseMove { .. }) {1183-                let now = Instant::now();1184-                if now.duration_since(last_absolute_mouse_time) < MOUSE_INTERVAL {1185-                    continue;1186-                }1187-                last_absolute_mouse_time = now;1188-            }1189-            if let Err(error) = write_client_input_event(1190-                &mut input_stream,1191-                &event,1192-                use_v1,1193-                Some(&input_send_diagnostics),1194-            )1195-            .await1196-            {1197-                eprintln!("[client] input stream write failed: {error}; disconnecting");1198-                break;1199-            }1200-        }1201-        while let Some(event) = pending_relative.take() {1202-            if let Err(error) = write_client_input_event(1203-                &mut input_stream,1204-                &event,1205-                use_v1,1206-                Some(&input_send_diagnostics),1207-            )1208-            .await1209-            {1210-                eprintln!("[client] final relative input write failed: {error}");1211-                break;1212-            }1213-        }1214-        let _ = input_stream.finish();1215-    });753+    tokio::spawn(run_input_forwarder(754+        input_send,755+        rx,756+        use_v1,757+        input_capabilities,758+        input_send_diagnostics,759+    ));1216760 1217761     // Frame reader — dual path: WebCodecs (raw bytes) or software JPEG decode1218762     let use_webcodecs = state.webcodecs.load(Ordering::SeqCst);@@ -2005,46 +1549,6 @@ mod tests {
20051549         active.store(false, Ordering::SeqCst);20061550     }20071551 2008-    #[test]2009-    fn pointer_feedback_channel_has_explicit_bounded_terminal_envelopes() {2010-        let position = serde_json::to_value(PointerFeedbackPayload::Position {2011-            sequence: 7,2012-            position: Some(PointerPosition { x: 1280, y: 800 }),2013-            pointer_visible: Some(false),2014-        })2015-        .unwrap();2016-        assert_eq!(position["type"], "position");2017-        assert_eq!(position["sequence"], 7);2018-        assert_eq!(position["position"]["x"], 1280);2019-        assert_eq!(position["pointer_visible"], false);2020-2021-        let legacy = serde_json::to_value(PointerFeedbackPayload::Position {2022-            sequence: 8,2023-            position: Some(PointerPosition { x: 640, y: 400 }),2024-            pointer_visible: None,2025-        })2026-        .unwrap();2027-        assert!(legacy.get("pointer_visible").is_none());2028-2029-        let eof = serde_json::to_value(PointerFeedbackPayload::Terminal {2030-            reason: PointerFeedbackTerminalReason::Eof,2031-        })2032-        .unwrap();2033-        assert_eq!(2034-            eof,2035-            serde_json::json!({ "type": "terminal", "reason": "eof" })2036-        );2037-2038-        let malformed = serde_json::to_value(PointerFeedbackPayload::Terminal {2039-            reason: PointerFeedbackTerminalReason::Malformed,2040-        })2041-        .unwrap();2042-        assert_eq!(2043-            malformed,2044-            serde_json::json!({ "type": "terminal", "reason": "malformed" })2045-        );2046-    }2047-20481552     #[test]20491553     fn sequence_checks_reject_duplicates_regressions_and_overflow() {20501554         assert_eq!(sequence_gap(41, 42).unwrap(), 0);@@ -2053,336 +1557,6 @@ mod tests {
20531557         assert!(sequence_gap(41, 40).is_err());20541558         assert!(sequence_gap(u64::MAX, 0).is_err());20551559     }2056-2057-    #[test]2058-    fn input_events_require_their_negotiated_capability() {2059-        let pointer = [Capability::AbsolutePointer];2060-        assert!(input_event_allowed(2061-            &pointer,2062-            &InputEvent::MouseMove { x: 1, y: 2 }2063-        ));2064-        assert!(input_event_allowed(2065-            &pointer,2066-            &InputEvent::MouseClick { b: 1 }2067-        ));2068-        assert!(input_event_allowed(2069-            &pointer,2070-            &InputEvent::MouseDown { b: 1 }2071-        ));2072-        assert!(input_event_allowed(&pointer, &InputEvent::MouseUp { b: 1 }));2073-        assert!(input_event_allowed(2074-            &pointer,2075-            &InputEvent::MouseScroll { dx: 0, dy: 1 }2076-        ));2077-        assert!(!input_event_allowed(2078-            &pointer,2079-            &InputEvent::MouseMoveRelative { dx: 1, dy: -2 }2080-        ));2081-        assert!(!input_event_allowed(2082-            &pointer,2083-            &InputEvent::MousePositionSync { x: 640, y: 400 }2084-        ));2085-        assert!(!input_event_allowed(2086-            &pointer,2087-            &InputEvent::KeyDown { k: "A".into() }2088-        ));2089-        assert!(!input_event_allowed(2090-            &pointer,2091-            &InputEvent::Text { s: "a".into() }2092-        ));2093-2094-        let relative_pointer = [Capability::RelativePointer];2095-        assert!(input_event_allowed(2096-            &relative_pointer,2097-            &InputEvent::MouseMoveRelative { dx: 1, dy: -2 }2098-        ));2099-        assert!(input_event_allowed(2100-            &relative_pointer,2101-            &InputEvent::MousePositionSync { x: 640, y: 400 }2102-        ));2103-        assert!(input_event_allowed(2104-            &relative_pointer,2105-            &InputEvent::MouseDown { b: 1 }2106-        ));2107-        assert!(input_event_allowed(2108-            &relative_pointer,2109-            &InputEvent::MouseUp { b: 1 }2110-        ));2111-        assert!(input_event_allowed(2112-            &relative_pointer,2113-            &InputEvent::MouseScroll { dx: 0, dy: 1 }2114-        ));2115-        assert!(!input_event_allowed(2116-            &relative_pointer,2117-            &InputEvent::MouseMove { x: 1, y: 2 }2118-        ));2119-2120-        let keyboard = [Capability::Keyboard];2121-        assert!(input_event_allowed(2122-            &keyboard,2123-            &InputEvent::KeyDown { k: "A".into() }2124-        ));2125-        assert!(input_event_allowed(2126-            &keyboard,2127-            &InputEvent::KeyUp { k: "A".into() }2128-        ));2129-        assert!(input_event_allowed(2130-            &keyboard,2131-            &InputEvent::KeyClick { k: "A".into() }2132-        ));2133-        assert!(!input_event_allowed(2134-            &keyboard,2135-            &InputEvent::Text { s: "a".into() }2136-        ));2137-2138-        let text = [Capability::Text];2139-        assert!(input_event_allowed(2140-            &text,2141-            &InputEvent::Text { s: "a".into() }2142-        ));2143-        assert!(!input_event_allowed(2144-            &text,2145-            &InputEvent::KeyDown { k: "A".into() }2146-        ));2147-2148-        let gamepad = [Capability::Gamepad];2149-        assert!(input_event_allowed(2150-            &gamepad,2151-            &InputEvent::Gamepad {2152-                state: sigil_protocol::GamepadState::default(),2153-            }2154-        ));2155-        assert!(!input_event_allowed(2156-            &keyboard,2157-            &InputEvent::Gamepad {2158-                state: sigil_protocol::GamepadState::default(),2159-            }2160-        ));2161-    }2162-2163-    #[test]2164-    fn empty_input_capabilities_are_view_only() {2165-        let capabilities = [];2166-        assert!(!input_event_allowed(2167-            &capabilities,2168-            &InputEvent::MouseMove { x: 1, y: 2 }2169-        ));2170-        assert!(!input_event_allowed(2171-            &capabilities,2172-            &InputEvent::KeyDown { k: "A".into() }2173-        ));2174-        assert!(!input_event_allowed(2175-            &capabilities,2176-            &InputEvent::Text { s: "a".into() }2177-        ));2178-        assert_eq!(2179-            InputAvailability::from_capabilities(&capabilities),2180-            InputAvailability {2181-                relative_pointer: false,2182-                pointer_position_feedback: false,2183-                absolute_pointer: false,2184-                keyboard: false,2185-                text: false,2186-                gamepad: false,2187-                input_ack: false,2188-                control: false,2189-            }2190-        );2191-    }2192-2193-    #[test]2194-    fn input_availability_reports_each_accepted_capability_exactly() {2195-        assert_eq!(2196-            InputAvailability::from_capabilities(&[Capability::AbsolutePointer, Capability::Text]),2197-            InputAvailability {2198-                relative_pointer: false,2199-                pointer_position_feedback: false,2200-                absolute_pointer: true,2201-                keyboard: false,2202-                text: true,2203-                gamepad: false,2204-                input_ack: false,2205-                control: true,2206-            }2207-        );2208-        assert_eq!(2209-            InputAvailability::from_capabilities(&[Capability::Keyboard]),2210-            InputAvailability {2211-                relative_pointer: false,2212-                pointer_position_feedback: false,2213-                absolute_pointer: false,2214-                keyboard: true,2215-                text: false,2216-                gamepad: false,2217-                input_ack: false,2218-                control: true,2219-            }2220-        );2221-        assert_eq!(2222-            InputAvailability::from_capabilities(&[Capability::Gamepad]),2223-            InputAvailability {2224-                relative_pointer: false,2225-                pointer_position_feedback: false,2226-                absolute_pointer: false,2227-                keyboard: false,2228-                text: false,2229-                gamepad: true,2230-                input_ack: false,2231-                control: true,2232-            }2233-        );2234-        assert!(2235-            InputAvailability::from_capabilities(&[Capability::Gamepad, Capability::InputAck])2236-                .input_ack2237-        );2238-        assert_eq!(2239-            InputAvailability::from_capabilities(&[2240-                Capability::RelativePointer,2241-                Capability::PointerPositionFeedback,2242-            ]),2243-            InputAvailability {2244-                relative_pointer: true,2245-                pointer_position_feedback: true,2246-                absolute_pointer: false,2247-                keyboard: false,2248-                text: false,2249-                gamepad: false,2250-                input_ack: false,2251-                control: true,2252-            }2253-        );2254-    }2255-2256-    #[test]2257-    fn pointer_feedback_without_input_ack_does_not_fail_ack_validation() {2258-        let diagnostics = StdMutex::new(NetworkSessionDiagnostics::new(Instant::now(), false));2259-        assert!(observe_input_ack_if_negotiated(&diagnostics, false, 0, Instant::now()).is_ok());2260-    }2261-2262-    #[test]2263-    fn input_capability_fallbacks_remove_only_one_protocol_extension_at_a_time() {2264-        let offers = input_capability_offers(InvitationGrants::ALL);2265-        assert_eq!(offers.len(), 8);2266-        let visibility = &offers[0];2267-        let position = &offers[1];2268-        let relative = &offers[2];2269-        let inherited = &offers[3];2270-2271-        assert!(2272-            offers[..4]2273-                .iter()2274-                .all(|offer| offer.contains(&Capability::InputAck))2275-        );2276-        assert!(2277-            offers[4..]2278-                .iter()2279-                .all(|offer| !offer.contains(&Capability::InputAck))2280-        );2281-        assert!(visibility.contains(&Capability::PointerVisibilityFeedback));2282-        assert!(visibility.contains(&Capability::PointerPositionFeedback));2283-        assert!(!position.contains(&Capability::PointerVisibilityFeedback));2284-        assert!(position.contains(&Capability::PointerPositionFeedback));2285-        assert!(!relative.contains(&Capability::PointerVisibilityFeedback));2286-        assert!(!relative.contains(&Capability::PointerPositionFeedback));2287-        assert!(relative.contains(&Capability::RelativePointer));2288-        assert!(!inherited.contains(&Capability::RelativePointer));2289-        assert_eq!(2290-            inherited.as_slice(),2291-            &[2292-                Capability::AbsolutePointer,2293-                Capability::Keyboard,2294-                Capability::Text,2295-                Capability::Gamepad,2296-                Capability::InputAck,2297-            ]2298-        );2299-    }2300-2301-    #[test]2302-    fn local_invitation_grants_bound_input_offers_before_the_host_intersection() {2303-        let view_only = input_capability_offers(InvitationGrants::VIEW);2304-        assert!(view_only.iter().all(Vec::is_empty));2305-2306-        let pointer = input_capability_offers(2307-            InvitationGrants::VIEW.union(InvitationGrants::POINTER_KEYBOARD),2308-        );2309-        assert!(pointer[0].contains(&Capability::Keyboard));2310-        assert!(pointer[0].contains(&Capability::InputAck));2311-        assert!(!pointer[0].contains(&Capability::Gamepad));2312-2313-        let gamepad =2314-            input_capability_offers(InvitationGrants::VIEW.union(InvitationGrants::GAMEPAD));2315-        assert_eq!(gamepad[0], vec![Capability::Gamepad, Capability::InputAck]);2316-        assert_eq!(gamepad[1], vec![Capability::Gamepad]);2317-    }2318-2319-    #[test]2320-    fn relative_pointer_accumulator_coalesces_chunks_and_resets() {2321-        let mut accumulator = RelativePointerAccumulator::default();2322-        accumulator.push(10, -20);2323-        accumulator.push(5, 8);2324-        assert_eq!(2325-            accumulator.take(),2326-            Some(InputEvent::MouseMoveRelative { dx: 15, dy: -12 })2327-        );2328-        assert_eq!(accumulator.take(), None);2329-2330-        accumulator.push(RELATIVE_POINTER_DELTA_MAX, RELATIVE_POINTER_DELTA_MIN);2331-        accumulator.push(1_000, -1_000);2332-        assert_eq!(2333-            accumulator.take(),2334-            Some(InputEvent::MouseMoveRelative {2335-                dx: RELATIVE_POINTER_DELTA_MAX,2336-                dy: RELATIVE_POINTER_DELTA_MIN,2337-            })2338-        );2339-        assert_eq!(2340-            accumulator.take(),2341-            Some(InputEvent::MouseMoveRelative {2342-                dx: 1_000,2343-                dy: -1_000,2344-            })2345-        );2346-        assert_eq!(accumulator.take(), None);2347-    }2348-2349-    #[test]2350-    fn relative_motion_chunks_are_staged_immediately_before_a_following_button() {2351-        let mut accumulator = RelativePointerAccumulator::default();2352-        assert_eq!(2353-            stage_relative_input(2354-                &mut accumulator,2355-                InputEvent::MouseMoveRelative {2356-                    dx: RELATIVE_POINTER_DELTA_MAX,2357-                    dy: RELATIVE_POINTER_DELTA_MIN,2358-                }2359-            ),2360-            None2361-        );2362-        assert_eq!(2363-            stage_relative_input(2364-                &mut accumulator,2365-                InputEvent::MouseMoveRelative { dx: 7, dy: -1 }2366-            ),2367-            None2368-        );2369-        assert_eq!(2370-            stage_relative_input(&mut accumulator, InputEvent::MouseDown { b: 1 }),2371-            Some(InputEvent::MouseDown { b: 1 })2372-        );2373-        assert_eq!(2374-            accumulator.take(),2375-            Some(InputEvent::MouseMoveRelative {2376-                dx: RELATIVE_POINTER_DELTA_MAX,2377-                dy: RELATIVE_POINTER_DELTA_MIN,2378-            })2379-        );2380-        assert_eq!(2381-            accumulator.take(),2382-            Some(InputEvent::MouseMoveRelative { dx: 7, dy: -1 })2383-        );2384-        assert_eq!(accumulator.take(), None);2385-    }23861560 }23871561 23881562 #[tauri::command]@@ -2390,19 +1564,7 @@ pub async fn iroh_client_send_input(
23901564     state: State<'_, AppState>,23911565     event: InputEvent,23921566 ) -> Result<bool, String> {2393-    let tx = state2394-        .input_send2395-        .lock()2396-        .await2397-        .clone()2398-        .ok_or_else(|| "Not connected to host".to_string())?;2399-    match tx.try_send(event) {2400-        Ok(()) => Ok(true),2401-        Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => Ok(false),2402-        Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => {2403-            Err("Input channel closed".to_string())2404-        }2405-    }1567+    send_input(&state, event).await24061568 }24071569 24081570 #[tauri::command]

M src-tauri/src/commands/state.rs +0 -2

@@ -176,10 +176,8 @@ fn reapply_hidden_cursor_rect(
176176 }177177 178178 pub const FRAME_ALPN: &[u8] = sigil_protocol::LEGACY_FRAME_ALPN_V0;179-pub const INPUT_ALPN: &[u8] = sigil_protocol::LEGACY_INPUT_ALPN_V0;180179 pub const RPID: &str = "sigil";181180 pub const SALT_MESSAGE: &str = "sigil-iroh-identity-v1";182-pub const CLIENT_INPUT_QUEUE_CAPACITY: usize = 256;183181 pub const AUDIO_DELIVERY_CAPACITY: usize = 3;184182 185183 #[derive(Debug, Default)]

A src-tauri/src/media/input_delivery.rs +919 -0

@@ -0,0 +1,919 @@
1+use std::sync::{Arc, Mutex as StdMutex};2+use std::time::{Duration, Instant};3+4+use iroh::Endpoint;5+use serde::Serialize;6+use sigil_protocol::{7+    Capability, INPUT_ALPN_V1, InputEvent, InvitationGrants, PointerPosition,8+    RELATIVE_POINTER_DELTA_MAX, RELATIVE_POINTER_DELTA_MIN, read_input_ack, write_input_event,9+};10+use tauri::ipc::Channel;11+12+use crate::commands::state::AppState;13+use crate::media::network_diagnostics::{NetworkSessionDiagnostics, lock_network_diagnostics};14+use crate::media::transport::{NegotiatedV1Stream, negotiate_v1};15+16+const INPUT_ALPN: &[u8] = sigil_protocol::LEGACY_INPUT_ALPN_V0;17+pub(crate) const CLIENT_INPUT_QUEUE_CAPACITY: usize = 256;18+19+#[derive(Debug, Clone, Copy, PartialEq, Eq)]20+pub(crate) struct InputAvailability {21+    pub(crate) relative_pointer: bool,22+    pub(crate) pointer_position_feedback: bool,23+    pub(crate) absolute_pointer: bool,24+    pub(crate) keyboard: bool,25+    pub(crate) text: bool,26+    pub(crate) gamepad: bool,27+    pub(crate) input_ack: bool,28+    pub(crate) control: bool,29+}30+31+impl InputAvailability {32+    fn from_capabilities(capabilities: &[Capability]) -> Self {33+        let relative_pointer = capabilities.contains(&Capability::RelativePointer);34+        let pointer_position_feedback = capabilities.contains(&Capability::PointerPositionFeedback);35+        let absolute_pointer = capabilities.contains(&Capability::AbsolutePointer);36+        let keyboard = capabilities.contains(&Capability::Keyboard);37+        let text = capabilities.contains(&Capability::Text);38+        let gamepad = capabilities.contains(&Capability::Gamepad);39+        let input_ack = capabilities.contains(&Capability::InputAck);40+        Self {41+            relative_pointer,42+            pointer_position_feedback,43+            absolute_pointer,44+            keyboard,45+            text,46+            gamepad,47+            input_ack,48+            control: relative_pointer || absolute_pointer || keyboard || text || gamepad,49+        }50+    }51+}52+53+#[derive(Serialize, Clone, Copy)]54+#[serde(tag = "type", rename_all = "snake_case")]55+pub enum PointerFeedbackPayload {56+    Position {57+        sequence: u64,58+        position: Option<PointerPosition>,59+        #[serde(skip_serializing_if = "Option::is_none")]60+        pointer_visible: Option<bool>,61+    },62+    Terminal {63+        reason: PointerFeedbackTerminalReason,64+    },65+}66+67+#[derive(Serialize, Clone, Copy)]68+#[serde(rename_all = "snake_case")]69+pub enum PointerFeedbackTerminalReason {70+    Eof,71+    Malformed,72+}73+74+#[derive(Debug, Default, PartialEq, Eq)]75+struct RelativePointerAccumulator {76+    dx: i64,77+    dy: i64,78+}79+80+impl RelativePointerAccumulator {81+    fn push(&mut self, dx: i32, dy: i32) {82+        self.dx = self.dx.saturating_add(i64::from(dx));83+        self.dy = self.dy.saturating_add(i64::from(dy));84+    }85+86+    fn take(&mut self) -> Option<InputEvent> {87+        if self.dx == 0 && self.dy == 0 {88+            return None;89+        }90+        let dx = self.dx.clamp(91+            i64::from(RELATIVE_POINTER_DELTA_MIN),92+            i64::from(RELATIVE_POINTER_DELTA_MAX),93+        ) as i32;94+        let dy = self.dy.clamp(95+            i64::from(RELATIVE_POINTER_DELTA_MIN),96+            i64::from(RELATIVE_POINTER_DELTA_MAX),97+        ) as i32;98+        let event = InputEvent::MouseMoveRelative { dx, dy };99+        self.dx -= i64::from(dx);100+        self.dy -= i64::from(dy);101+        Some(event)102+    }103+104+    fn is_pending(&self) -> bool {105+        self.dx != 0 || self.dy != 0106+    }107+}108+109+fn stage_relative_input(110+    pending: &mut RelativePointerAccumulator,111+    event: InputEvent,112+) -> Option<InputEvent> {113+    match event {114+        InputEvent::MouseMoveRelative { dx, dy } => {115+            pending.push(dx, dy);116+            None117+        }118+        event => Some(event),119+    }120+}121+122+async fn open_negotiated_input_stream(123+    endpoint: &Endpoint,124+    address: &iroh::EndpointAddr,125+    nonce: [u8; 16],126+    capabilities: Vec<Capability>,127+) -> Result<128+    (129+        iroh::endpoint::Connection,130+        iroh::endpoint::SendStream,131+        iroh::endpoint::RecvStream,132+        NegotiatedV1Stream,133+    ),134+    String,135+> {136+    let connection = endpoint137+        .connect(address.clone(), INPUT_ALPN_V1)138+        .await139+        .map_err(|error| format!("Failed to connect input stream: {error}"))?;140+    let (mut send, mut recv) = connection141+        .open_bi()142+        .await143+        .map_err(|error| format!("Failed to open input stream: {error}"))?;144+    let negotiation = negotiate_v1(145+        &mut send,146+        &mut recv,147+        nonce,148+        capabilities,149+        None,150+        "input",151+        None,152+    )153+    .await?;154+    Ok((connection, send, recv, negotiation))155+}156+157+fn input_capability_offers(grants: InvitationGrants) -> Vec<Vec<Capability>> {158+    let base = [159+        vec![160+            Capability::RelativePointer,161+            Capability::PointerPositionFeedback,162+            Capability::PointerVisibilityFeedback,163+            Capability::AbsolutePointer,164+            Capability::Keyboard,165+            Capability::Text,166+            Capability::Gamepad,167+        ],168+        vec![169+            Capability::RelativePointer,170+            Capability::PointerPositionFeedback,171+            Capability::AbsolutePointer,172+            Capability::Keyboard,173+            Capability::Text,174+            Capability::Gamepad,175+        ],176+        vec![177+            Capability::RelativePointer,178+            Capability::AbsolutePointer,179+            Capability::Keyboard,180+            Capability::Text,181+            Capability::Gamepad,182+        ],183+        vec![184+            Capability::AbsolutePointer,185+            Capability::Keyboard,186+            Capability::Text,187+            Capability::Gamepad,188+        ],189+    ];190+    let has_input_grant = grants.contains(InvitationGrants::POINTER_KEYBOARD)191+        || grants.contains(InvitationGrants::GAMEPAD);192+    let mut offers = Vec::with_capacity(base.len() * 2);193+    if has_input_grant {194+        for mut offer in base.clone() {195+            offer.push(Capability::InputAck);196+            offers.push(offer);197+        }198+    }199+    offers.extend(base);200+    for offer in &mut offers {201+        offer.retain(|capability| match capability {202+            Capability::Gamepad => grants.contains(InvitationGrants::GAMEPAD),203+            Capability::AbsolutePointer204+            | Capability::RelativePointer205+            | Capability::Keyboard206+            | Capability::Text207+            | Capability::PointerPositionFeedback208+            | Capability::PointerVisibilityFeedback => {209+                grants.contains(InvitationGrants::POINTER_KEYBOARD)210+            }211+            Capability::InputAck => has_input_grant,212+            _ => true,213+        });214+    }215+    offers.dedup();216+    offers217+}218+219+fn input_event_allowed(capabilities: &[Capability], event: &InputEvent) -> bool {220+    match event {221+        InputEvent::Probe => capabilities.contains(&Capability::InputAck),222+        InputEvent::MouseMove { .. } => capabilities.contains(&Capability::AbsolutePointer),223+        InputEvent::MouseMoveRelative { .. } => capabilities.contains(&Capability::RelativePointer),224+        InputEvent::MousePositionSync { .. } => capabilities.contains(&Capability::RelativePointer),225+        InputEvent::MouseClick { .. }226+        | InputEvent::MouseDown { .. }227+        | InputEvent::MouseUp { .. }228+        | InputEvent::MouseScroll { .. } => {229+            capabilities.contains(&Capability::RelativePointer)230+                || capabilities.contains(&Capability::AbsolutePointer)231+        }232+        InputEvent::KeyDown { .. } | InputEvent::KeyUp { .. } | InputEvent::KeyClick { .. } => {233+            capabilities.contains(&Capability::Keyboard)234+        }235+        InputEvent::Text { .. } => capabilities.contains(&Capability::Text),236+        InputEvent::Gamepad { .. } => capabilities.contains(&Capability::Gamepad),237+    }238+}239+240+async fn write_client_input_event(241+    stream: &mut iroh::endpoint::SendStream,242+    event: &InputEvent,243+    use_v1: bool,244+    diagnostics: Option<&Arc<StdMutex<NetworkSessionDiagnostics>>>,245+) -> Result<(), String> {246+    if let Some(diagnostics) = diagnostics {247+        lock_network_diagnostics(diagnostics).begin_input_send(Instant::now());248+    }249+    if use_v1 {250+        write_input_event(stream, event)251+            .await252+            .map_err(|error| error.to_string())253+    } else {254+        let json = serde_json::to_string(event).map_err(|error| error.to_string())?;255+        stream256+            .write_all(format!("{json}\n").as_bytes())257+            .await258+            .map_err(|error| error.to_string())259+    }260+}261+262+fn observe_input_ack_if_negotiated(263+    diagnostics: &StdMutex<NetworkSessionDiagnostics>,264+    negotiated: bool,265+    sequence: u64,266+    now: Instant,267+) -> Result<(), String> {268+    if !negotiated {269+        return Ok(());270+    }271+    lock_network_diagnostics(diagnostics).observe_input_ack(sequence, now)272+}273+274+pub(crate) struct InputSession {275+    pub(crate) connection: iroh::endpoint::Connection,276+    pub(crate) send: iroh::endpoint::SendStream,277+    pub(crate) recv: iroh::endpoint::RecvStream,278+    pub(crate) capabilities: Vec<Capability>,279+    pub(crate) availability: InputAvailability,280+}281+282+pub(crate) async fn open_input_session(283+    endpoint: &Endpoint,284+    address: &iroh::EndpointAddr,285+    nonce: [u8; 16],286+    media_session_id: Option<u64>,287+    grants: InvitationGrants,288+    use_v1: bool,289+) -> Result<InputSession, String> {290+    let input_alpn = if use_v1 { INPUT_ALPN_V1 } else { INPUT_ALPN };291+    let (connection, send, recv, capabilities) = if use_v1 {292+        let mut errors = Vec::new();293+        let mut accepted = None;294+        // Older hosts reject unknown capability enum values. Try all four295+        // pointer feature levels with ACK first, then repeat the exact legacy296+        // offers without ACK so lack of diagnostics never forces absolute297+        // pointer input prematurely.298+        for capabilities in input_capability_offers(grants) {299+            match open_negotiated_input_stream(endpoint, address, nonce, capabilities).await {300+                Ok(result) => {301+                    accepted = Some(result);302+                    break;303+                }304+                Err(error) => errors.push(error),305+            }306+        }307+        let (connection, send, recv, input_negotiation) = accepted308+            .ok_or_else(|| format!("All input capability offers failed: {}", errors.join("; ")))?;309+        if Some(input_negotiation.session_id) != media_session_id {310+            return Err("Host returned mismatched media and input sessions".to_string());311+        }312+        (connection, send, recv, input_negotiation.capabilities)313+    } else {314+        let input_conn = endpoint315+            .connect(address.clone(), input_alpn)316+            .await317+            .map_err(|e| format!("Failed to connect input stream: {}", e))?;318+        let (mut send, recv) = input_conn319+            .open_bi()320+            .await321+            .map_err(|e| format!("Failed to open input stream: {}", e))?;322+        send.write_all(&[1u8])323+            .await324+            .map_err(|e| format!("Failed to send input start: {}", e))?;325+        // The inherited protocol predates negotiation and supports the current326+        // absolute-pointer, keyboard, and text event set.327+        (328+            input_conn,329+            send,330+            recv,331+            vec![332+                Capability::AbsolutePointer,333+                Capability::Keyboard,334+                Capability::Text,335+            ],336+        )337+    };338+    let availability = InputAvailability::from_capabilities(&capabilities);339+    Ok(InputSession {340+        connection,341+        send,342+        recv,343+        capabilities,344+        availability,345+    })346+}347+348+pub(crate) async fn run_input_feedback(349+    mut input_feedback: iroh::endpoint::RecvStream,350+    feedback_diagnostics: Arc<StdMutex<NetworkSessionDiagnostics>>,351+    mut pointer_feedback_enabled: bool,352+    input_ack_enabled: bool,353+    pointer_channel: Channel<PointerFeedbackPayload>,354+) {355+    let terminal_reason = loop {356+        let response = match read_input_ack(&mut input_feedback).await {357+            Ok(Some(response)) => response,358+            Ok(None) => {359+                lock_network_diagnostics(&feedback_diagnostics).mark_input_feedback_closed();360+                break PointerFeedbackTerminalReason::Eof;361+            }362+            Err(error) => {363+                lock_network_diagnostics(&feedback_diagnostics).mark_input_feedback_malformed();364+                eprintln!("[client] invalid input feedback: {error}");365+                break PointerFeedbackTerminalReason::Malformed;366+            }367+        };368+        if let Err(error) = observe_input_ack_if_negotiated(369+            &feedback_diagnostics,370+            input_ack_enabled,371+            response.sequence,372+            Instant::now(),373+        ) {374+            eprintln!("[client] invalid input acknowledgement: {error}");375+            break PointerFeedbackTerminalReason::Malformed;376+        }377+        if pointer_feedback_enabled378+            && pointer_channel379+                .send(PointerFeedbackPayload::Position {380+                    sequence: response.sequence,381+                    position: response.pointer_position,382+                    pointer_visible: response.pointer_visible,383+                })384+                .is_err()385+        {386+            // Losing the webview's pointer channel must not stop ACK387+            // draining and apply backpressure to host input.388+            pointer_feedback_enabled = false;389+            if !input_ack_enabled {390+                return;391+            }392+        }393+    };394+    // The session-owned channel emits at most one terminal message.395+    // JavaScript rejects deliveries from superseded channel closures.396+    let _ = pointer_channel.send(PointerFeedbackPayload::Terminal {397+        reason: terminal_reason,398+    });399+}400+401+pub(crate) async fn run_input_forwarder(402+    mut input_stream: iroh::endpoint::SendStream,403+    rx: tokio::sync::mpsc::Receiver<InputEvent>,404+    use_v1: bool,405+    input_capabilities: Vec<Capability>,406+    input_send_diagnostics: Arc<StdMutex<NetworkSessionDiagnostics>>,407+) {408+    let mut rx = rx;409+    const MOUSE_INTERVAL: Duration = Duration::from_millis(16);410+    let started = Instant::now();411+    let mut last_absolute_mouse_time = started.checked_sub(MOUSE_INTERVAL).unwrap_or(started);412+    let mut last_relative_mouse_time = started.checked_sub(MOUSE_INTERVAL).unwrap_or(started);413+    let mut pending_relative = RelativePointerAccumulator::default();414+    let mut input_open = true;415+416+    while input_open {417+        let event = if pending_relative.is_pending() {418+            let wait = MOUSE_INTERVAL.saturating_sub(last_relative_mouse_time.elapsed());419+            if wait.is_zero() {420+                let Some(event) = pending_relative.take() else {421+                    continue;422+                };423+                if let Err(error) = write_client_input_event(424+                    &mut input_stream,425+                    &event,426+                    use_v1,427+                    Some(&input_send_diagnostics),428+                )429+                .await430+                {431+                    eprintln!("[client] input stream write failed: {error}; disconnecting");432+                    break;433+                }434+                last_relative_mouse_time = Instant::now();435+                continue;436+            }437+            tokio::select! {438+                event = rx.recv() => event,439+                () = tokio::time::sleep(wait) => {440+                    let Some(event) = pending_relative.take() else {441+                        continue;442+                    };443+                    if let Err(error) = write_client_input_event(444+                        &mut input_stream,445+                        &event,446+                        use_v1,447+                        Some(&input_send_diagnostics),448+                    ).await {449+                        eprintln!("[client] input stream write failed: {error}; disconnecting");450+                        break;451+                    }452+                    last_relative_mouse_time = Instant::now();453+                    continue;454+                }455+            }456+        } else {457+            rx.recv().await458+        };459+        let Some(event) = event else {460+            input_open = false;461+            continue;462+        };463+        // The host's accepted capability set is an authorization boundary.464+        // Drop unavailable event classes silently so event contents never465+        // reach logs or the wire even if a compromised webview invokes the466+        // command directly.467+        if !input_event_allowed(&input_capabilities, &event) {468+            continue;469+        }470+        let Some(event) = stage_relative_input(&mut pending_relative, event) else {471+            continue;472+        };473+        let mut flushed_relative_barrier = false;474+        while let Some(relative_barrier) = pending_relative.take() {475+            if let Err(error) = write_client_input_event(476+                &mut input_stream,477+                &relative_barrier,478+                use_v1,479+                Some(&input_send_diagnostics),480+            )481+            .await482+            {483+                eprintln!("[client] input stream write failed: {error}; disconnecting");484+                input_open = false;485+                break;486+            }487+            flushed_relative_barrier = true;488+        }489+        if !input_open {490+            break;491+        }492+        if flushed_relative_barrier {493+            last_relative_mouse_time = Instant::now();494+        }495+        if matches!(event, InputEvent::MouseMove { .. }) {496+            let now = Instant::now();497+            if now.duration_since(last_absolute_mouse_time) < MOUSE_INTERVAL {498+                continue;499+            }500+            last_absolute_mouse_time = now;501+        }502+        if let Err(error) = write_client_input_event(503+            &mut input_stream,504+            &event,505+            use_v1,506+            Some(&input_send_diagnostics),507+        )508+        .await509+        {510+            eprintln!("[client] input stream write failed: {error}; disconnecting");511+            break;512+        }513+    }514+    while let Some(event) = pending_relative.take() {515+        if let Err(error) = write_client_input_event(516+            &mut input_stream,517+            &event,518+            use_v1,519+            Some(&input_send_diagnostics),520+        )521+        .await522+        {523+            eprintln!("[client] final relative input write failed: {error}");524+            break;525+        }526+    }527+    let _ = input_stream.finish();528+}529+530+pub(crate) async fn send_input(state: &AppState, event: InputEvent) -> Result<bool, String> {531+    let tx = state532+        .input_send533+        .lock()534+        .await535+        .clone()536+        .ok_or_else(|| "Not connected to host".to_string())?;537+    match tx.try_send(event) {538+        Ok(()) => Ok(true),539+        Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => Ok(false),540+        Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => {541+            Err("Input channel closed".to_string())542+        }543+    }544+}545+546+#[cfg(test)]547+mod tests {548+    use super::*;549+550+    #[test]551+    fn pointer_feedback_channel_has_explicit_bounded_terminal_envelopes() {552+        let position = serde_json::to_value(PointerFeedbackPayload::Position {553+            sequence: 7,554+            position: Some(PointerPosition { x: 1280, y: 800 }),555+            pointer_visible: Some(false),556+        })557+        .unwrap();558+        assert_eq!(position["type"], "position");559+        assert_eq!(position["sequence"], 7);560+        assert_eq!(position["position"]["x"], 1280);561+        assert_eq!(position["pointer_visible"], false);562+563+        let legacy = serde_json::to_value(PointerFeedbackPayload::Position {564+            sequence: 8,565+            position: Some(PointerPosition { x: 640, y: 400 }),566+            pointer_visible: None,567+        })568+        .unwrap();569+        assert!(legacy.get("pointer_visible").is_none());570+571+        let eof = serde_json::to_value(PointerFeedbackPayload::Terminal {572+            reason: PointerFeedbackTerminalReason::Eof,573+        })574+        .unwrap();575+        assert_eq!(576+            eof,577+            serde_json::json!({ "type": "terminal", "reason": "eof" })578+        );579+580+        let malformed = serde_json::to_value(PointerFeedbackPayload::Terminal {581+            reason: PointerFeedbackTerminalReason::Malformed,582+        })583+        .unwrap();584+        assert_eq!(585+            malformed,586+            serde_json::json!({ "type": "terminal", "reason": "malformed" })587+        );588+    }589+590+    #[test]591+    fn input_events_require_their_negotiated_capability() {592+        let pointer = [Capability::AbsolutePointer];593+        assert!(input_event_allowed(594+            &pointer,595+            &InputEvent::MouseMove { x: 1, y: 2 }596+        ));597+        assert!(input_event_allowed(598+            &pointer,599+            &InputEvent::MouseClick { b: 1 }600+        ));601+        assert!(input_event_allowed(602+            &pointer,603+            &InputEvent::MouseDown { b: 1 }604+        ));605+        assert!(input_event_allowed(&pointer, &InputEvent::MouseUp { b: 1 }));606+        assert!(input_event_allowed(607+            &pointer,608+            &InputEvent::MouseScroll { dx: 0, dy: 1 }609+        ));610+        assert!(!input_event_allowed(611+            &pointer,612+            &InputEvent::MouseMoveRelative { dx: 1, dy: -2 }613+        ));614+        assert!(!input_event_allowed(615+            &pointer,616+            &InputEvent::MousePositionSync { x: 640, y: 400 }617+        ));618+        assert!(!input_event_allowed(619+            &pointer,620+            &InputEvent::KeyDown { k: "A".into() }621+        ));622+        assert!(!input_event_allowed(623+            &pointer,624+            &InputEvent::Text { s: "a".into() }625+        ));626+627+        let relative_pointer = [Capability::RelativePointer];628+        assert!(input_event_allowed(629+            &relative_pointer,630+            &InputEvent::MouseMoveRelative { dx: 1, dy: -2 }631+        ));632+        assert!(input_event_allowed(633+            &relative_pointer,634+            &InputEvent::MousePositionSync { x: 640, y: 400 }635+        ));636+        assert!(input_event_allowed(637+            &relative_pointer,638+            &InputEvent::MouseDown { b: 1 }639+        ));640+        assert!(input_event_allowed(641+            &relative_pointer,642+            &InputEvent::MouseUp { b: 1 }643+        ));644+        assert!(input_event_allowed(645+            &relative_pointer,646+            &InputEvent::MouseScroll { dx: 0, dy: 1 }647+        ));648+        assert!(!input_event_allowed(649+            &relative_pointer,650+            &InputEvent::MouseMove { x: 1, y: 2 }651+        ));652+653+        let keyboard = [Capability::Keyboard];654+        assert!(input_event_allowed(655+            &keyboard,656+            &InputEvent::KeyDown { k: "A".into() }657+        ));658+        assert!(input_event_allowed(659+            &keyboard,660+            &InputEvent::KeyUp { k: "A".into() }661+        ));662+        assert!(input_event_allowed(663+            &keyboard,664+            &InputEvent::KeyClick { k: "A".into() }665+        ));666+        assert!(!input_event_allowed(667+            &keyboard,668+            &InputEvent::Text { s: "a".into() }669+        ));670+671+        let text = [Capability::Text];672+        assert!(input_event_allowed(673+            &text,674+            &InputEvent::Text { s: "a".into() }675+        ));676+        assert!(!input_event_allowed(677+            &text,678+            &InputEvent::KeyDown { k: "A".into() }679+        ));680+681+        let gamepad = [Capability::Gamepad];682+        assert!(input_event_allowed(683+            &gamepad,684+            &InputEvent::Gamepad {685+                state: sigil_protocol::GamepadState::default(),686+            }687+        ));688+        assert!(!input_event_allowed(689+            &keyboard,690+            &InputEvent::Gamepad {691+                state: sigil_protocol::GamepadState::default(),692+            }693+        ));694+    }695+696+    #[test]697+    fn empty_input_capabilities_are_view_only() {698+        let capabilities = [];699+        assert!(!input_event_allowed(700+            &capabilities,701+            &InputEvent::MouseMove { x: 1, y: 2 }702+        ));703+        assert!(!input_event_allowed(704+            &capabilities,705+            &InputEvent::KeyDown { k: "A".into() }706+        ));707+        assert!(!input_event_allowed(708+            &capabilities,709+            &InputEvent::Text { s: "a".into() }710+        ));711+        assert_eq!(712+            InputAvailability::from_capabilities(&capabilities),713+            InputAvailability {714+                relative_pointer: false,715+                pointer_position_feedback: false,716+                absolute_pointer: false,717+                keyboard: false,718+                text: false,719+                gamepad: false,720+                input_ack: false,721+                control: false,722+            }723+        );724+    }725+726+    #[test]727+    fn input_availability_reports_each_accepted_capability_exactly() {728+        assert_eq!(729+            InputAvailability::from_capabilities(&[Capability::AbsolutePointer, Capability::Text]),730+            InputAvailability {731+                relative_pointer: false,732+                pointer_position_feedback: false,733+                absolute_pointer: true,734+                keyboard: false,735+                text: true,736+                gamepad: false,737+                input_ack: false,738+                control: true,739+            }740+        );741+        assert_eq!(742+            InputAvailability::from_capabilities(&[Capability::Keyboard]),743+            InputAvailability {744+                relative_pointer: false,745+                pointer_position_feedback: false,746+                absolute_pointer: false,747+                keyboard: true,748+                text: false,749+                gamepad: false,750+                input_ack: false,751+                control: true,752+            }753+        );754+        assert_eq!(755+            InputAvailability::from_capabilities(&[Capability::Gamepad]),756+            InputAvailability {757+                relative_pointer: false,758+                pointer_position_feedback: false,759+                absolute_pointer: false,760+                keyboard: false,761+                text: false,762+                gamepad: true,763+                input_ack: false,764+                control: true,765+            }766+        );767+        assert!(768+            InputAvailability::from_capabilities(&[Capability::Gamepad, Capability::InputAck])769+                .input_ack770+        );771+        assert_eq!(772+            InputAvailability::from_capabilities(&[773+                Capability::RelativePointer,774+                Capability::PointerPositionFeedback,775+            ]),776+            InputAvailability {777+                relative_pointer: true,778+                pointer_position_feedback: true,779+                absolute_pointer: false,780+                keyboard: false,781+                text: false,782+                gamepad: false,783+                input_ack: false,784+                control: true,785+            }786+        );787+    }788+789+    #[test]790+    fn pointer_feedback_without_input_ack_does_not_fail_ack_validation() {791+        let diagnostics = StdMutex::new(NetworkSessionDiagnostics::new(Instant::now(), false));792+        assert!(observe_input_ack_if_negotiated(&diagnostics, false, 0, Instant::now()).is_ok());793+    }794+795+    #[test]796+    fn input_capability_fallbacks_remove_only_one_protocol_extension_at_a_time() {797+        let offers = input_capability_offers(InvitationGrants::ALL);798+        assert_eq!(offers.len(), 8);799+        let visibility = &offers[0];800+        let position = &offers[1];801+        let relative = &offers[2];802+        let inherited = &offers[3];803+804+        assert!(805+            offers[..4]806+                .iter()807+                .all(|offer| offer.contains(&Capability::InputAck))808+        );809+        assert!(810+            offers[4..]811+                .iter()812+                .all(|offer| !offer.contains(&Capability::InputAck))813+        );814+        assert!(visibility.contains(&Capability::PointerVisibilityFeedback));815+        assert!(visibility.contains(&Capability::PointerPositionFeedback));816+        assert!(!position.contains(&Capability::PointerVisibilityFeedback));817+        assert!(position.contains(&Capability::PointerPositionFeedback));818+        assert!(!relative.contains(&Capability::PointerVisibilityFeedback));819+        assert!(!relative.contains(&Capability::PointerPositionFeedback));820+        assert!(relative.contains(&Capability::RelativePointer));821+        assert!(!inherited.contains(&Capability::RelativePointer));822+        assert_eq!(823+            inherited.as_slice(),824+            &[825+                Capability::AbsolutePointer,826+                Capability::Keyboard,827+                Capability::Text,828+                Capability::Gamepad,829+                Capability::InputAck,830+            ]831+        );832+    }833+834+    #[test]835+    fn local_invitation_grants_bound_input_offers_before_the_host_intersection() {836+        let view_only = input_capability_offers(InvitationGrants::VIEW);837+        assert!(view_only.iter().all(Vec::is_empty));838+839+        let pointer = input_capability_offers(840+            InvitationGrants::VIEW.union(InvitationGrants::POINTER_KEYBOARD),841+        );842+        assert!(pointer[0].contains(&Capability::Keyboard));843+        assert!(pointer[0].contains(&Capability::InputAck));844+        assert!(!pointer[0].contains(&Capability::Gamepad));845+846+        let gamepad =847+            input_capability_offers(InvitationGrants::VIEW.union(InvitationGrants::GAMEPAD));848+        assert_eq!(gamepad[0], vec![Capability::Gamepad, Capability::InputAck]);849+        assert_eq!(gamepad[1], vec![Capability::Gamepad]);850+    }851+852+    #[test]853+    fn relative_pointer_accumulator_coalesces_chunks_and_resets() {854+        let mut accumulator = RelativePointerAccumulator::default();855+        accumulator.push(10, -20);856+        accumulator.push(5, 8);857+        assert_eq!(858+            accumulator.take(),859+            Some(InputEvent::MouseMoveRelative { dx: 15, dy: -12 })860+        );861+        assert_eq!(accumulator.take(), None);862+863+        accumulator.push(RELATIVE_POINTER_DELTA_MAX, RELATIVE_POINTER_DELTA_MIN);864+        accumulator.push(1_000, -1_000);865+        assert_eq!(866+            accumulator.take(),867+            Some(InputEvent::MouseMoveRelative {868+                dx: RELATIVE_POINTER_DELTA_MAX,869+                dy: RELATIVE_POINTER_DELTA_MIN,870+            })871+        );872+        assert_eq!(873+            accumulator.take(),874+            Some(InputEvent::MouseMoveRelative {875+                dx: 1_000,876+                dy: -1_000,877+            })878+        );879+        assert_eq!(accumulator.take(), None);880+    }881+882+    #[test]883+    fn relative_motion_chunks_are_staged_immediately_before_a_following_button() {884+        let mut accumulator = RelativePointerAccumulator::default();885+        assert_eq!(886+            stage_relative_input(887+                &mut accumulator,888+                InputEvent::MouseMoveRelative {889+                    dx: RELATIVE_POINTER_DELTA_MAX,890+                    dy: RELATIVE_POINTER_DELTA_MIN,891+                }892+            ),893+            None894+        );895+        assert_eq!(896+            stage_relative_input(897+                &mut accumulator,898+                InputEvent::MouseMoveRelative { dx: 7, dy: -1 }899+            ),900+            None901+        );902+        assert_eq!(903+            stage_relative_input(&mut accumulator, InputEvent::MouseDown { b: 1 }),904+            Some(InputEvent::MouseDown { b: 1 })905+        );906+        assert_eq!(907+            accumulator.take(),908+            Some(InputEvent::MouseMoveRelative {909+                dx: RELATIVE_POINTER_DELTA_MAX,910+                dy: RELATIVE_POINTER_DELTA_MIN,911+            })912+        );913+        assert_eq!(914+            accumulator.take(),915+            Some(InputEvent::MouseMoveRelative { dx: 7, dy: -1 })916+        );917+        assert_eq!(accumulator.take(), None);918+    }919+}

M src-tauri/src/media/mod.rs +1 -0

@@ -1,6 +1,7 @@
11 pub(crate) mod adaptive_feedback;22 pub(crate) mod audio_delivery;33 pub(crate) mod frame_channel;4+pub(crate) mod input_delivery;45 pub(crate) mod metrics;56 pub(crate) mod moq_catalog;67 pub(crate) mod moq_receiver;