refactor(portal): split client media modules
- commit
- c917330cfcac5012ce9478a1ff2d04f56f023756
- parent
- 78cce98bb09542f3a8758a8e2c9e779d4029a5b7
- parent
- bc8f8b60de00154f0fea0fe80b8d634e69d9a842
- author
- FelineStateMachine <dami.etoile@gmail.com>
@@ -1,2876 +1,24 @@
1-use super::auth::derive_iroh_secret_from_key;2-use super::enrollment::{connection_enrollment, mark_invitation_redeemed};3-use super::moq_catalog::subscribe_goq_video_track;4-5-use super::network_diagnostics::PathMode;6-use super::network_diagnostics::{7- NetworkDiagnosticsSnapshot, NetworkLeg, NetworkSessionDiagnostics,8-};9-use super::state::{10- AUDIO_DELIVERY_CAPACITY, AccumulatedMediaFeedback, AppState, AudioDeliveryState,11- CLIENT_INPUT_QUEUE_CAPACITY, FRAME_ALPN, INPUT_ALPN, MediaFeedbackSender,12- development_direct_node_available,13-};14-use base64::Engine;15-use iroh::{Endpoint, SecretKey, endpoint::presets};16-use iroh_moq::{Moq, MoqSession};17-18-use moq_net::Track;19-use moq_net::{BroadcastConsumer, GroupConsumer, TrackConsumer};20-use openh264::{formats::YUVSource, nal_units};21-use serde::{Deserialize, Serialize};22-23-use sigil_protocol::MOQ_VIDEO_H264_TRACK;24-use sigil_protocol::{25- AUDIO_ALPN_V1, AdaptiveBitrateDecisionV1, AdaptiveBitrateReasonFlagsV1, AdaptiveBitrateStateV1,26- AudioFlags, AudioPacket, AudioPacketHeader, CONTROL_ALPN_V1, Capability, ClientHello,27- FrameFlags, INPUT_ALPN_V1, InputEvent, InvitationGrants, KeyframeRequestReasonV3,28- MAX_MEDIA_GROUP_BYTES_V3, MAX_MEDIA_OBJECT_DELIVERY_TIMEOUT_MS, MAX_MEDIA_OBJECT_ID_V3,29- MAX_MEDIA_PAYLOAD_LEN, MAX_VIDEO_DIMENSION, MAX_VIDEO_PIXELS, MEDIA_ALPN_V1, MEDIA_ALPN_V2,30- MEDIA_ALPN_V3, MEDIA_FEEDBACK_ALPN_V1, MediaCodec, MediaControlRequestV3, MediaFeedbackFlags,31- MediaFeedbackReportV1, MediaFrame, MediaObjectV3, PointerPosition, PointerSurfaceDimensions,32- ProtocolError, RELATIVE_POINTER_DELTA_MAX, RELATIVE_POINTER_DELTA_MIN,33- decode_media_frame_object, media_moq_broadcast_name, read_adaptive_bitrate_decision_v1,34- read_host_hello, read_input_ack, read_media_frame, read_media_object, read_media_object_v3,35- write_client_hello, write_input_event, write_media_control_request_v3,36- write_media_feedback_report_v1,37-};38-use std::collections::{BTreeMap, VecDeque};39-use std::io::Cursor;40-use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};41-use std::sync::{Arc, Mutex as StdMutex, MutexGuard as StdMutexGuard};42-use std::time::{Duration, Instant};1+use super::state::AppState;2+pub use crate::media::adaptive_feedback::ClientMediaFeedbackReport;3+use crate::media::adaptive_feedback::send_media_feedback;4+use crate::media::audio_delivery;5+pub use crate::media::connect::ConnectResult;6+use crate::media::connect::{connect_client, disconnect_client};7+8+pub use crate::media::frame_channel::FramePayload;9+use crate::media::frame_channel::acknowledge_frame_delivery;10+use crate::media::input_delivery::send_input;11+12+pub use crate::media::input_delivery::{PointerFeedbackPayload, PointerFeedbackTerminalReason};13+use crate::media::media_control::request_keyframe;14+use sigil_protocol::InputEvent;4315 use tauri::{44- AppHandle, Emitter, Manager, State,16+ AppHandle, State,4517 ipc::{Channel, Response},4618 };4719 4820 // ─── Client commands ──────────────────────────────────────────────────────────4921 50-fn byte_to_codec(value: u8) -> &'static str {51- match value {52- 1 => "h264",53- 2 => "h265",54- 3 => "av1",55- _ => "h264",56- }57-}58-59-60-pub struct FramePayload {61- pub generation: u64,62- pub width: u32,63- pub height: u32,64- pub data: String,65- pub keyframe: bool,66- pub codec: String,67- pub capture_timestamp_micros: Option<u64>,68- pub pts_micros: Option<i64>,69- pub discontinuity: bool,70-}71-72-73-struct FrameErrorPayload {74- generation: u64,75- error: String,76-}77-78-fn emit_frame_error(app: &AppHandle, generation: u64, error: impl Into<String>) {79- let _ = app.emit(80- "frame-error",81- FrameErrorPayload {82- generation,83- error: error.into(),84- },85- );86-}87-88-const LEGACY_MEDIA_IDLE_TIMEOUT: Duration = Duration::from_secs(10);89-const CLIENT_MEDIA_OBJECT_CAPACITY: usize = 4;90-const CLIENT_MEDIA_OBJECT_READ_TIMEOUT: Duration = Duration::from_secs(1);91-const CLIENT_MOQ_CONNECT_TIMEOUT: Duration = Duration::from_secs(10);92-const CLIENT_MOQ_SUBSCRIBE_TIMEOUT: Duration = Duration::from_secs(10);93-// Match the protocol's maximum single-object delivery horizon. A publisher may94-// never hold a partially delivered object open indefinitely.95-const CLIENT_MOQ_OBJECT_READ_TIMEOUT: Duration =96- Duration::from_millis(MAX_MEDIA_OBJECT_DELIVERY_TIMEOUT_MS as u64);97-// Sigil's external encoder can take 500 ms to reach its next configured IDR.98-// Allow another 500 ms for a relay path to deliver the superseding group.99-const CLIENT_MOQ_GROUP_RECOVERY_TIMEOUT: Duration = Duration::from_secs(1);100-// Absorb brief webview→Rust acknowledgment jitter without allowing Tauri IPC101-// to grow without bound. Four 60 fps frames cap this handoff at about 67 ms;102-// WebCodecs has a separate, stricter decode-queue bound in the frontend.103-const CLIENT_FRAME_CHANNEL_CAPACITY: usize = 4;104-// Three 20 ms Opus packets cap the Rust→webview handoff at 60 ms. The105-// AudioWorklet owns a separate fixed ring and never feeds back into transport.106-const CLIENT_FRAME_STATS_INTERVAL: Duration = Duration::from_millis(250);107-const CLIENT_ENDPOINT_CLOSE_TIMEOUT: Duration = Duration::from_secs(2);108-const CLIENT_MEDIA_FEEDBACK_CONNECT_TIMEOUT: Duration = Duration::from_secs(3);109-const CLIENT_MEDIA_FEEDBACK_IO_TIMEOUT: Duration = Duration::from_secs(5);110-const CLIENT_ADAPTIVE_DECISION_DELIVERY_INTERVAL: Duration = Duration::from_secs(1);111-const CLIENT_FRAME_RATE_WINDOW: Duration = Duration::from_secs(1);112-const CLIENT_FRAME_TIMING_WINDOW: Duration = Duration::from_secs(5);113-// Host configuration permits at most 240 fps. Leave a little headroom for114-// timer-boundary samples while keeping every rate window strictly bounded.115-const CLIENT_FRAME_RATE_SAMPLE_CAPACITY: usize = 256;116-const CLIENT_FRAME_TIMING_SAMPLE_CAPACITY: usize = 512;117-const FRAME_CHANNEL_MAGIC: [u8; 4] = *b"SGFR";118-const FRAME_CHANNEL_VERSION: u8 = 1;119-const FRAME_CHANNEL_HEADER_LEN: usize = 40;120-const FRAME_CHANNEL_FLAG_KEYFRAME: u8 = 1 << 0;121-const FRAME_CHANNEL_FLAG_DISCONTINUITY: u8 = 1 << 1;122-const FRAME_CHANNEL_FLAG_CODEC_CONFIG: u8 = 1 << 2;123-const FRAME_CHANNEL_OPTIONAL_U64_NONE: u64 = u64::MAX;124-const FRAME_CHANNEL_OPTIONAL_I64_NONE: i64 = i64::MIN;125-const AUDIO_CHANNEL_MAGIC: [u8; 4] = *b"SGAC";126-const AUDIO_CHANNEL_VERSION: u16 = 1;127-const AUDIO_CHANNEL_HEADER_LEN: usize = 24;128-129-130-struct RollingFrameRate {131- samples: VecDeque<Duration>,132-}133-134-impl RollingFrameRate {135- fn record(&mut self, elapsed: Duration) {136- self.prune(elapsed);137- if self.samples.len() == CLIENT_FRAME_RATE_SAMPLE_CAPACITY {138- self.samples.pop_front();139- }140- self.samples.push_back(elapsed);141- }142-143- fn rate(&mut self, elapsed: Duration) -> f64 {144- self.prune(elapsed);145- let observed = elapsed.min(CLIENT_FRAME_RATE_WINDOW).as_secs_f64();146- if observed <= f64::EPSILON {147- return 0.0;148- }149- self.samples.len() as f64 / observed150- }151-152- fn prune(&mut self, elapsed: Duration) {153- let cutoff = elapsed.saturating_sub(CLIENT_FRAME_RATE_WINDOW);154- while self155- .samples156- .front()157- .is_some_and(|sample| *sample <= cutoff && elapsed >= CLIENT_FRAME_RATE_WINDOW)158- {159- self.samples.pop_front();160- }161- }162-}163-164-165-struct TimedDurationSample {166- observed_at: Duration,167- duration: Duration,168-}169-170-171-struct DurationWindowSummary {172- sample_count: usize,173- p50_ms: Option<f64>,174- p95_ms: Option<f64>,175- max_ms: Option<f64>,176-}177-178-179-struct RollingDurationWindow {180- samples: VecDeque<TimedDurationSample>,181-}182-183-impl RollingDurationWindow {184- fn record(&mut self, observed_at: Duration, duration: Duration) {185- self.prune(observed_at);186- if self.samples.len() == CLIENT_FRAME_TIMING_SAMPLE_CAPACITY {187- self.samples.pop_front();188- }189- self.samples.push_back(TimedDurationSample {190- observed_at,191- duration,192- });193- }194-195- fn summary(&mut self, elapsed: Duration) -> DurationWindowSummary {196- self.prune(elapsed);197- if self.samples.is_empty() {198- return DurationWindowSummary::default();199- }200-201- let mut durations = self202- .samples203- .iter()204- .map(|sample| sample.duration)205- .collect::<Vec<_>>();206- durations.sort_unstable();207- DurationWindowSummary {208- sample_count: durations.len(),209- p50_ms: Some(duration_ms(nearest_rank(&durations, 50))),210- p95_ms: Some(duration_ms(nearest_rank(&durations, 95))),211- max_ms: durations.last().copied().map(duration_ms),212- }213- }214-215- fn prune(&mut self, elapsed: Duration) {216- let cutoff = elapsed.saturating_sub(CLIENT_FRAME_TIMING_WINDOW);217- while self.samples.front().is_some_and(|sample| {218- sample.observed_at <= cutoff && elapsed >= CLIENT_FRAME_TIMING_WINDOW219- }) {220- self.samples.pop_front();221- }222- }223-}224-225-fn nearest_rank(values: &[Duration], percentile: usize) -> Duration {226- debug_assert!(!values.is_empty());227- debug_assert!((1..=100).contains(&percentile));228- let rank = values.len().saturating_mul(percentile).div_ceil(100);229- values[rank.saturating_sub(1).min(values.len() - 1)]230-}231-232-fn duration_ms(duration: Duration) -> f64 {233- duration.as_secs_f64() * 1_000.0234-}235-236-237-struct ClientMediaMetrics {238- transport_rate: RollingFrameRate,239- frontend_send_rate: RollingFrameRate,240- transport_intervals: RollingDurationWindow,241- frontend_ipc_send_durations: RollingDurationWindow,242- transport_received_total: u64,243- frontend_sent_total: u64,244- sequence_dropped_total: u64,245- transport_object_dropped_total: u64,246- transport_late_object_dropped_total: u64,247- frontend_queue_dropped_total: u64,248- frontend_resync_dropped_total: u64,249- frontend_queue_peak: usize,250- frontend_resync_episode_total: u64,251- frontend_resync_started_at: Option<Duration>,252- frontend_resync_completed_duration: Duration,253- frontend_resync_max_duration: Duration,254- last_transport_received_at: Option<Duration>,255- last_sequence: Option<u64>,256- last_keyframe: bool,257-}258-259-impl ClientMediaMetrics {260- fn observe_transport_receive(261- &mut self,262- elapsed: Duration,263- sequence: Option<u64>,264- keyframe: bool,265- ) {266- if let Some(previous) = self.last_transport_received_at {267- let interval = elapsed.saturating_sub(previous);268- // Gamescope capture is damage-driven. A gap longer than the269- // rolling diagnostics window is idle, not one giant hitch; start270- // a fresh cadence anchor while retaining the exact boundary.271- if interval <= CLIENT_FRAME_TIMING_WINDOW {272- self.transport_intervals.record(elapsed, interval);273- }274- }275- self.last_transport_received_at = Some(elapsed);276- self.transport_received_total = self.transport_received_total.saturating_add(1);277- self.transport_rate.record(elapsed);278- self.last_sequence = sequence;279- self.last_keyframe = keyframe;280- }281-282- fn observe_frontend_send(&mut self, elapsed: Duration) {283- self.frontend_sent_total = self.frontend_sent_total.saturating_add(1);284- self.frontend_send_rate.record(elapsed);285- }286-287- fn observe_sequence_drop(&mut self, count: u64) {288- self.sequence_dropped_total = self.sequence_dropped_total.saturating_add(count);289- }290-291- fn observe_transport_object_drop(&mut self, late: bool) {292- self.transport_object_dropped_total = self.transport_object_dropped_total.saturating_add(1);293- if late {294- self.transport_late_object_dropped_total =295- self.transport_late_object_dropped_total.saturating_add(1);296- }297- }298-299- fn observe_frontend_queue_drop(&mut self) {300- self.frontend_queue_dropped_total = self.frontend_queue_dropped_total.saturating_add(1);301- }302-303- fn observe_frontend_resync_drop(&mut self) {304- self.frontend_resync_dropped_total = self.frontend_resync_dropped_total.saturating_add(1);305- }306-307- fn begin_frontend_resync(&mut self, elapsed: Duration) {308- if self.frontend_resync_started_at.is_none() {309- self.frontend_resync_episode_total =310- self.frontend_resync_episode_total.saturating_add(1);311- self.frontend_resync_started_at = Some(elapsed);312- }313- }314-315- fn finish_frontend_resync(&mut self, elapsed: Duration) {316- let Some(started_at) = self.frontend_resync_started_at.take() else {317- return;318- };319- let duration = elapsed.saturating_sub(started_at);320- self.frontend_resync_completed_duration = self321- .frontend_resync_completed_duration322- .saturating_add(duration);323- self.frontend_resync_max_duration = self.frontend_resync_max_duration.max(duration);324- }325-326- fn observe_frontend_ipc_send_duration(&mut self, elapsed: Duration, duration: Duration) {327- self.frontend_ipc_send_durations.record(elapsed, duration);328- }329-330- fn observe_frontend_queue_depth(&mut self, depth: usize) {331- self.frontend_queue_peak = self.frontend_queue_peak.max(depth);332- }333-334- fn snapshot(335- &mut self,336- elapsed: Duration,337- frontend_queue_depth: usize,338- network_diagnostics: NetworkDiagnosticsSnapshot,339- generation: u64,340- ) -> FrameStatsPayload {341- let transport_receive_fps = self.transport_rate.rate(elapsed);342- let frontend_send_fps = self.frontend_send_rate.rate(elapsed);343- let frontend_dropped_total = self344- .frontend_queue_dropped_total345- .saturating_add(self.frontend_resync_dropped_total);346- let transport_intervals = self.transport_intervals.summary(elapsed);347- let frontend_ipc_send_durations = self.frontend_ipc_send_durations.summary(elapsed);348- let frontend_resync_current_duration = self349- .frontend_resync_started_at350- .map(|started_at| elapsed.saturating_sub(started_at));351- let frontend_resync_duration = self352- .frontend_resync_completed_duration353- .saturating_add(frontend_resync_current_duration.unwrap_or_default());354- let frontend_resync_max_duration = self355- .frontend_resync_max_duration356- .max(frontend_resync_current_duration.unwrap_or_default());357- FrameStatsPayload {358- generation,359- stats_version: 4,360- transport_receive_fps,361- frontend_send_fps,362- transport_received_total: self.transport_received_total,363- frontend_sent_total: self.frontend_sent_total,364- sequence_dropped_total: self.sequence_dropped_total,365- transport_object_dropped_total: self.transport_object_dropped_total,366- transport_late_object_dropped_total: self.transport_late_object_dropped_total,367- frontend_queue_dropped_total: self.frontend_queue_dropped_total,368- frontend_resync_dropped_total: self.frontend_resync_dropped_total,369- frontend_dropped_total,370- frontend_queue_depth,371- frontend_queue_peak: self.frontend_queue_peak,372- frontend_queue_capacity: CLIENT_FRAME_CHANNEL_CAPACITY,373- frontend_resync_episode_total: self.frontend_resync_episode_total,374- frontend_resync_active: self.frontend_resync_started_at.is_some(),375- frontend_resync_duration_ms_total: duration_ms(frontend_resync_duration),376- frontend_resync_duration_ms_current: frontend_resync_current_duration.map(duration_ms),377- frontend_resync_duration_ms_max: duration_ms(frontend_resync_max_duration),378- timing_window_ms: duration_ms(CLIENT_FRAME_TIMING_WINDOW),379- timing_sample_capacity: CLIENT_FRAME_TIMING_SAMPLE_CAPACITY,380- transport_interval_sample_count: transport_intervals.sample_count,381- transport_interval_p50_ms: transport_intervals.p50_ms,382- transport_interval_p95_ms: transport_intervals.p95_ms,383- transport_interval_max_ms: transport_intervals.max_ms,384- frontend_ipc_send_duration_sample_count: frontend_ipc_send_durations.sample_count,385- frontend_ipc_send_duration_p50_ms: frontend_ipc_send_durations.p50_ms,386- frontend_ipc_send_duration_p95_ms: frontend_ipc_send_durations.p95_ms,387- frontend_ipc_send_duration_max_ms: frontend_ipc_send_durations.max_ms,388- path_mode: network_diagnostics.media.mode.as_str(),389- path_rtt_ms: network_diagnostics.media.rtt_current_ms,390- network_diagnostics,391- sequence: self.last_sequence,392- keyframe: self.last_keyframe,393- // Compatibility aliases for the currently bundled frontend. New394- // diagnostics should consume the explicitly named fields above.395- fps: frontend_send_fps,396- transport_fps: transport_receive_fps,397- frontend_fps: frontend_send_fps,398- count: self.frontend_sent_total,399- host_dropped_frames: self.sequence_dropped_total,400- frontend_dropped_frames: frontend_dropped_total,401- }402- }403-}404-405-fn lock_client_media_metrics(406- metrics: &StdMutex<ClientMediaMetrics>,407-) -> StdMutexGuard<'_, ClientMediaMetrics> {408- metrics409- .lock()410- .unwrap_or_else(std::sync::PoisonError::into_inner)411-}412-413-fn lock_network_diagnostics(414- diagnostics: &StdMutex<NetworkSessionDiagnostics>,415-) -> StdMutexGuard<'_, NetworkSessionDiagnostics> {416- diagnostics417- .lock()418- .unwrap_or_else(std::sync::PoisonError::into_inner)419-}420-421-fn close_generation_connection<T>(422- connection: Option<(u64, T)>,423- close: impl FnOnce(T),424-) -> Option<u64> {425- connection.map(|(generation, connection)| {426- close(connection);427- generation428- })429-}430-431-fn take_generation_owned<T>(slot: &mut Option<(u64, T)>, expected_generation: u64) -> Option<T> {432- if slot433- .as_ref()434- .is_some_and(|(generation, _)| *generation == expected_generation)435- {436- slot.take().map(|(_, value)| value)437- } else {438- None439- }440-}441-442-fn take_generation_owned_triple<T, U>(443- slot: &mut Option<(u64, T, U)>,444- expected_generation: u64,445-) -> Option<(T, U)> {446- if slot447- .as_ref()448- .is_some_and(|(generation, _, _)| *generation == expected_generation)449- {450- slot.take().map(|(_, value, companion)| (value, companion))451- } else {452- None453- }454-}455-456-async fn retire_media_feedback_generation(app: &AppHandle, generation: u64) -> bool {457- let state = app.state::<AppState>();458- let connection = {459- let _connection_serial = state.client_connection_serial.lock().await;460- let mut slot = state.media_feedback.lock().await;461- take_generation_owned_triple(&mut slot, generation).map(|(connection, _)| connection)462- };463- let Some(connection) = connection else {464- return false;465- };466- connection.close(0_u32.into(), b"adaptive feedback ended");467- true468-}469-470-async fn retire_upstream_moq_generation(471- app: &AppHandle,472- media_generation: u64,473- audio_generation: Option<u64>,474-) -> bool {475- let state = app.state::<AppState>();476- let endpoint = {477- // Selection and retirement are one generation-checked transaction.478- // A stale reader must never close a replacement session whose task was479- // installed after an explicit disconnect/reconnect.480- let _connection_serial = state.client_connection_serial.lock().await;481- let media_connection = {482- let mut slot = state.media_connection.lock().await;483- take_generation_owned(&mut slot, media_generation)484- };485- let Some(media_connection) = media_connection else {486- return false;487- };488-489- {490- let mut control = state.media_control.lock().await;491- let _ = take_generation_owned(&mut control, media_generation);492- }493- {494- let mut delivery = state.frame_delivery.lock().await;495- let _ = take_generation_owned(&mut delivery, media_generation);496- }497- let feedback_connection = {498- let mut feedback = state.media_feedback.lock().await;499- take_generation_owned_triple(&mut feedback, media_generation)500- .map(|(connection, _)| connection)501- };502- *state.input_send.lock().await = None;503-504- let audio_connection = if let Some(audio_generation) = audio_generation {505- if let Err(error) = cancel_audio_generation(506- &state.audio_connection_generation,507- &state.audio_deliveries,508- audio_generation,509- ) {510- eprintln!(511- "[client] failed to retire audio generation after upstream MoQ ended: {error}"512- );513- }514- let mut slot = state.audio_connection.lock().await;515- take_generation_owned(&mut slot, audio_generation)516- } else {517- None518- };519-520- let endpoint = state.client_endpoint.lock().await.take();521- media_connection.close(0_u32.into(), b"upstream MoQ media ended");522- if let Some(feedback_connection) = feedback_connection {523- feedback_connection.close(0_u32.into(), b"upstream MoQ media ended");524- }525- if let Some(audio_connection) = audio_connection {526- audio_connection.close(0_u32.into(), b"upstream MoQ media ended");527- }528- state529- .client_connection_active530- .store(false, Ordering::SeqCst);531- endpoint532- };533-534- if let Some(endpoint) = endpoint535- && tokio::time::timeout(CLIENT_ENDPOINT_CLOSE_TIMEOUT, endpoint.close())536- .await537- .is_err()538- {539- eprintln!("[client] timed out retiring endpoint after upstream MoQ media ended");540- }541- true542-}543-544-545-struct FrameStatsPayload {546- generation: u64,547- stats_version: u8,548- transport_receive_fps: f64,549- frontend_send_fps: f64,550- transport_received_total: u64,551- frontend_sent_total: u64,552- sequence_dropped_total: u64,553- transport_object_dropped_total: u64,554- transport_late_object_dropped_total: u64,555- frontend_queue_dropped_total: u64,556- frontend_resync_dropped_total: u64,557- frontend_dropped_total: u64,558- frontend_queue_depth: usize,559- frontend_queue_peak: usize,560- frontend_queue_capacity: usize,561- frontend_resync_episode_total: u64,562- frontend_resync_active: bool,563- frontend_resync_duration_ms_total: f64,564- frontend_resync_duration_ms_current: Option<f64>,565- frontend_resync_duration_ms_max: f64,566- timing_window_ms: f64,567- timing_sample_capacity: usize,568- transport_interval_sample_count: usize,569- transport_interval_p50_ms: Option<f64>,570- transport_interval_p95_ms: Option<f64>,571- transport_interval_max_ms: Option<f64>,572- frontend_ipc_send_duration_sample_count: usize,573- frontend_ipc_send_duration_p50_ms: Option<f64>,574- frontend_ipc_send_duration_p95_ms: Option<f64>,575- frontend_ipc_send_duration_max_ms: Option<f64>,576- path_mode: &'static str,577- path_rtt_ms: Option<f64>,578- network_diagnostics: NetworkDiagnosticsSnapshot,579- sequence: Option<u64>,580- keyframe: bool,581- fps: f64,582- transport_fps: f64,583- frontend_fps: f64,584- count: u64,585- host_dropped_frames: u64,586- frontend_dropped_frames: u64,587-}588-589-struct FrameEnvelopeMetadata<'a> {590- width: u32,591- height: u32,592- codec: &'a str,593- keyframe: bool,594- discontinuity: bool,595- codec_config: bool,596- sequence: Option<u64>,597- capture_timestamp_micros: Option<u64>,598- pts_micros: Option<i64>,599-}600-601-fn encode_frame_envelope(602- metadata: FrameEnvelopeMetadata<'_>,603- payload: &[u8],604-) -> Result<Vec<u8>, String> {605- validate_legacy_media_header(metadata.width, metadata.height, payload.len())?;606- if metadata.codec_config && !metadata.keyframe {607- return Err("Frame codec configuration requires a keyframe".to_string());608- }609- if metadata.sequence == Some(FRAME_CHANNEL_OPTIONAL_U64_NONE) {610- return Err("Frame sequence collides with the channel sentinel".to_string());611- }612- if metadata.capture_timestamp_micros == Some(FRAME_CHANNEL_OPTIONAL_U64_NONE) {613- return Err("Capture timestamp collides with the channel sentinel".to_string());614- }615- if metadata.pts_micros == Some(FRAME_CHANNEL_OPTIONAL_I64_NONE) {616- return Err("Frame PTS collides with the channel sentinel".to_string());617- }618- let width = u16::try_from(metadata.width).map_err(|_| {619- format!(620- "Frame width does not fit channel envelope: {}",621- metadata.width622- )623- })?;624- let height = u16::try_from(metadata.height).map_err(|_| {625- format!(626- "Frame height does not fit channel envelope: {}",627- metadata.height628- )629- })?;630- let payload_len = u32::try_from(payload.len()).map_err(|_| {631- format!(632- "Frame payload does not fit channel envelope: {}",633- payload.len()634- )635- })?;636- let codec = match metadata.codec {637- "h264" => 1,638- "h265" => 2,639- "av1" => 3,640- other => return Err(format!("Unsupported frame channel codec: {other}")),641- };642- let mut flags = 0_u8;643- if metadata.keyframe {644- flags |= FRAME_CHANNEL_FLAG_KEYFRAME;645- }646- if metadata.discontinuity {647- flags |= FRAME_CHANNEL_FLAG_DISCONTINUITY;648- }649- if metadata.codec_config {650- flags |= FRAME_CHANNEL_FLAG_CODEC_CONFIG;651- }652-653- let mut envelope = Vec::with_capacity(FRAME_CHANNEL_HEADER_LEN + payload.len());654- envelope.extend_from_slice(&FRAME_CHANNEL_MAGIC);655- envelope.push(FRAME_CHANNEL_VERSION);656- envelope.push(codec);657- envelope.push(flags);658- envelope.push(0); // Reserved; the parser rejects non-zero values.659- envelope.extend_from_slice(&width.to_be_bytes());660- envelope.extend_from_slice(&height.to_be_bytes());661- envelope.extend_from_slice(&payload_len.to_be_bytes());662- envelope.extend_from_slice(663- &metadata664- .sequence665- .unwrap_or(FRAME_CHANNEL_OPTIONAL_U64_NONE)666- .to_be_bytes(),667- );668- envelope.extend_from_slice(669- &metadata670- .capture_timestamp_micros671- .unwrap_or(FRAME_CHANNEL_OPTIONAL_U64_NONE)672- .to_be_bytes(),673- );674- envelope.extend_from_slice(675- &metadata676- .pts_micros677- .unwrap_or(FRAME_CHANNEL_OPTIONAL_I64_NONE)678- .to_be_bytes(),679- );680- envelope.extend_from_slice(payload);681- debug_assert_eq!(envelope.len(), FRAME_CHANNEL_HEADER_LEN + payload.len());682- Ok(envelope)683-}684-685-fn try_reserve_frame_channel_slot(in_flight: &AtomicUsize) -> bool {686- in_flight687- .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |current| {688- (current < CLIENT_FRAME_CHANNEL_CAPACITY).then_some(current + 1)689- })690- .is_ok()691-}692-693-fn release_frame_channel_slot(in_flight: &AtomicUsize) {694- let _ = in_flight.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |current| {695- current.checked_sub(1)696- });697-}698-699-fn release_frame_channel_slot_for_generation(700- in_flight: &AtomicUsize,701- current_generation: u64,702- generation: u64,703-) -> bool {704- if generation == 0 || current_generation != generation {705- return false;706- }707- release_frame_channel_slot(in_flight);708- true709-}710-711-fn lock_audio_deliveries(712- deliveries: &StdMutex<AudioDeliveryState>,713-) -> StdMutexGuard<'_, AudioDeliveryState> {714- deliveries715- .lock()716- .unwrap_or_else(std::sync::PoisonError::into_inner)717-}718-719-fn next_audio_generation(counter: &AtomicU64) -> Result<u64, String> {720- counter721- .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |current| {722- current.checked_add(1)723- })724- .map(|previous| previous + 1)725- .map_err(|_| "Audio connection generation overflowed".to_string())726-}727-728-fn next_media_generation(counter: &AtomicU64) -> Result<u64, String> {729- counter730- .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |current| {731- current.checked_add(1)732- })733- .map(|previous| previous + 1)734- .map_err(|_| "Media connection generation overflowed".to_string())735-}736-737-fn audio_generation_is_current(counter: &AtomicU64, generation: u64) -> bool {738- counter.load(Ordering::SeqCst) == generation739-}740-741-fn emit_audio_event_if_current(742- app: &AppHandle,743- event: &str,744- generation_counter: &AtomicU64,745- deliveries: &StdMutex<AudioDeliveryState>,746- generation: u64,747- payload: impl FnOnce(&AudioDeliveryState) -> serde_json::Value,748-) -> bool {749- let deliveries = lock_audio_deliveries(deliveries);750- if generation_counter.load(Ordering::SeqCst) != generation751- || deliveries.generation() != Some(generation)752- {753- return false;754- }755- let _ = app.emit(event, payload(&deliveries));756- true757-}758-759-struct ClientConnectGuard {760- active: Arc<AtomicBool>,761- committed: bool,762-}763-764-impl ClientConnectGuard {765- fn acquire(active: Arc<AtomicBool>) -> Result<Self, String> {766- active767- .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)768- .map_err(|_| "A client connection is already active or in progress".to_string())?;769- Ok(Self {770- active,771- committed: false,772- })773- }774-775- fn commit(mut self) {776- self.committed = true;777- }778-}779-780-impl Drop for ClientConnectGuard {781- fn drop(&mut self) {782- if !self.committed {783- self.active.store(false, Ordering::SeqCst);784- }785- }786-}787-788-789-pub struct ConnectResult {790- pub connected: bool,791- pub host_node_id: Option<String>,792- pub development_mode: bool,793- pub media_transport: &'static str,794- pub pointer_surface_dimensions: Option<PointerSurfaceDimensions>,795- pub relative_pointer_available: bool,796- pub pointer_position_feedback_available: bool,797- pub absolute_pointer_available: bool,798- pub keyboard_available: bool,799- pub text_available: bool,800- pub gamepad_available: bool,801- pub control_available: bool,802- pub audio_available: bool,803- pub audio_generation: Option<u64>,804- pub audio_error: Option<String>,805- pub media_generation: u64,806- pub adaptive_feedback_available: bool,807- pub adaptive_feedback_error: Option<String>,808- pub error: Option<String>,809-}810-811-812-pub struct ClientMediaFeedbackReport {813- pub interval_ms: u16,814- pub last_sequence: Option<u64>,815- pub transport_dropped_delta: u32,816- pub frontend_dropped_delta: u32,817- pub decoder_dropped_delta: u32,818- pub presenter_dropped_delta: u32,819- pub frontend_queue_depth: u8,820- pub frontend_queue_capacity: u8,821- pub decode_queue_depth: u8,822- pub decode_queue_capacity: u8,823- pub presenter_queue_depth: u8,824- pub presenter_queue_capacity: u8,825- pub transport_delivery_p95_ms: Option<f64>,826- pub decode_p95_ms: Option<f64>,827- pub presentation_p95_ms: Option<f64>,828- pub resync_active: bool,829-}830-831-832-struct AdaptiveBitrateDecisionPayload {833- generation: u64,834- decision: AdaptiveBitrateDecisionDiagnostic,835-}836-837-838-struct AdaptiveBitrateDecisionDiagnostic {839- decision_id: u64,840- report_id: u64,841- target_kbps: u32,842- floor_kbps: u32,843- ceiling_kbps: u32,844- state: &'static str,845- reasons: Vec<&'static str>,846- applied: bool,847-}848-849-struct NegotiatedV1Stream {850- session_id: u64,851- capabilities: Vec<Capability>,852- pointer_surface_dimensions: Option<PointerSurfaceDimensions>,853-}854-855-856-enum MediaTransport {857- UpstreamMoq,858- LegacyV0,859- ReliableStreamV1,860- IndependentObjectsV2,861- GroupedObjectsV3,862-}863-864-impl MediaTransport {865- const fn diagnostic_name(self) -> &'static str {866- match self {867- Self::UpstreamMoq => "iroh-moq",868- Self::LegacyV0 => "reliable-v0",869- Self::ReliableStreamV1 => "reliable-v1",870- Self::IndependentObjectsV2 => "independent-v2",871- Self::GroupedObjectsV3 => "grouped-v3",872- }873- }874-875- const fn supports_adaptive_feedback(self) -> bool {876- matches!(self, Self::UpstreamMoq | Self::GroupedObjectsV3)877- }878-}879-880-881-enum MoqMediaReadOutcome {882- Frame {883- frame: MediaFrame,884- discontinuity: bool,885- },886- Dropped {887- reason: KeyframeRequestReasonV3,888- },889- Malformed(String),890-}891-892-893-enum MoqGroupRecovery {894- /// Sigil deliberately cancels the old GOP when a configured IDR starts a895- /// replacement. This is normal live-edge supersession, not evidence that896- /// another keyframe request is needed.897- ExpectedSupersession,898- RecoverableGap(KeyframeRequestReasonV3),899-}900-901-struct MoqGroupCursor {902- group: GroupConsumer,903- sequence: u64,904- object_count: usize,905- object_bytes: usize,906- group_gap: bool,907- replacement_for_cancelled_group: bool,908-}909-910-/// Own every upstream handle for as long as the Portal frame task is alive.911-/// In particular, dropping `Moq` aborts its actor and dropping the consumers912-/// cancels their subscriptions, so none of these may be scoped only to setup.913-struct MoqMediaLifetime {914- _moq: Moq,915- _session: MoqSession,916- _broadcast: BroadcastConsumer,917-}918-919-struct MoqMediaReceiver {920- _lifetime: Option<MoqMediaLifetime>,921- track: TrackConsumer,922- current_group: Option<MoqGroupCursor>,923- last_group_sequence: Option<u64>,924- last_frame_sequence: Option<u64>,925- waiting_for_keyframe: bool,926- pending_group_recovery: Option<MoqGroupRecovery>,927-}928-929-impl MoqMediaReceiver {930- fn new(931- moq: Moq,932- session: MoqSession,933- broadcast: BroadcastConsumer,934- track: TrackConsumer,935- ) -> Self {936- Self {937- _lifetime: Some(MoqMediaLifetime {938- _moq: moq,939- _session: session,940- _broadcast: broadcast,941- }),942- track,943- current_group: None,944- last_group_sequence: None,945- last_frame_sequence: None,946- waiting_for_keyframe: true,947- pending_group_recovery: None,948- }949- }950-951- 952- fn for_test(track: TrackConsumer) -> Self {953- Self {954- _lifetime: None,955- track,956- current_group: None,957- last_group_sequence: None,958- last_frame_sequence: None,959- waiting_for_keyframe: true,960- pending_group_recovery: None,961- }962- }963-964- async fn next(&mut self) -> Result<Option<MoqMediaReadOutcome>, String> {965- self.next_with_timeouts(966- CLIENT_MOQ_OBJECT_READ_TIMEOUT,967- CLIENT_MOQ_GROUP_RECOVERY_TIMEOUT,968- )969- .await970- }971-972- async fn next_with_timeouts(973- &mut self,974- object_read_timeout: Duration,975- group_recovery_timeout: Duration,976- ) -> Result<Option<MoqMediaReadOutcome>, String> {977- loop {978- if self.current_group.is_none() {979- let replacement_for_cancelled_group = self.pending_group_recovery.is_some();980- let group = if replacement_for_cancelled_group {981- match tokio::time::timeout(982- group_recovery_timeout,983- self.track.next_group(),984- )985- .await986- {987- Ok(group) => group,988- Err(_) => {989- let recovery = self990- .pending_group_recovery991- .take()992- .expect("pending MoQ recovery reason was present");993- return match recovery {994- MoqGroupRecovery::ExpectedSupersession => Err(format!(995- "Timed out after {} ms waiting for the MoQ group that supersedes an expected GOP cancellation",996- group_recovery_timeout.as_millis()997- )),998- MoqGroupRecovery::RecoverableGap(reason) => {999- Ok(Some(MoqMediaReadOutcome::Dropped { reason }))1000- }1001- };1002- }1003- }1004- } else {1005- self.track.next_group().await1006- }1007- .map_err(|error| format!("Upstream MoQ video track failed: {error}"))?;1008- let Some(group) = group else {1009- return Ok(None);1010- };1011- self.pending_group_recovery = None;1012- let sequence = group.sequence;1013- let group_gap =1014- match classify_moq_group_sequence(self.last_group_sequence, sequence) {1015- Ok(group_gap) => group_gap,1016- Err(error) => return Ok(Some(MoqMediaReadOutcome::Malformed(error))),1017- };1018- if group_gap {1019- self.waiting_for_keyframe = true;1020- }1021- self.current_group = Some(MoqGroupCursor {1022- group,1023- sequence,1024- object_count: 0,1025- object_bytes: 0,1026- group_gap,1027- replacement_for_cancelled_group,1028- });1029- }1030-1031- let cursor = self1032- .current_group1033- .as_mut()1034- .expect("MoQ group cursor was initialized");1035- let object =1036- match tokio::time::timeout(object_read_timeout, cursor.group.read_frame()).await {1037- Err(_) => {1038- let sequence = cursor.sequence;1039- self.last_group_sequence = Some(sequence);1040- self.current_group = None;1041- self.waiting_for_keyframe = true;1042- return Ok(Some(MoqMediaReadOutcome::Dropped {1043- reason: KeyframeRequestReasonV3::DeliveryTimeout,1044- }));1045- }1046- Ok(Ok(object)) => object,1047- Ok(Err(error)) if moq_group_error_is_recoverable(&error) => {1048- let sequence = cursor.sequence;1049- self.last_group_sequence = Some(sequence);1050- self.current_group = None;1051- self.waiting_for_keyframe = true;1052- let reason = moq_group_error_reason(&error);1053- self.pending_group_recovery =1054- Some(if matches!(error, moq_net::Error::Cancel) {1055- MoqGroupRecovery::ExpectedSupersession1056- } else {1057- MoqGroupRecovery::RecoverableGap(reason)1058- });1059- continue;1060- }1061- Ok(Err(error)) => {1062- return Ok(Some(MoqMediaReadOutcome::Malformed(format!(1063- "Upstream MoQ group {} failed: {error}",1064- cursor.sequence1065- ))));1066- }1067- };1068- let Some(object) = object else {1069- if cursor.object_count == 0 {1070- if cursor.replacement_for_cancelled_group {1071- self.last_group_sequence = Some(cursor.sequence);1072- self.current_group = None;1073- self.waiting_for_keyframe = true;1074- return Ok(Some(MoqMediaReadOutcome::Dropped {1075- reason: KeyframeRequestReasonV3::TransportGap,1076- }));1077- }1078- return Ok(Some(MoqMediaReadOutcome::Malformed(format!(1079- "Upstream MoQ group {} was empty",1080- cursor.sequence1081- ))));1082- }1083- self.last_group_sequence = Some(cursor.sequence);1084- self.current_group = None;1085- continue;1086- };1087-1088- let next_group_bytes = match validate_moq_object_bounds(1089- cursor.sequence,1090- cursor.object_count,1091- cursor.object_bytes,1092- object.len(),1093- ) {1094- Ok(next_group_bytes) => next_group_bytes,1095- Err(error) => return Ok(Some(MoqMediaReadOutcome::Malformed(error))),1096- };1097- let frame = match decode_media_frame_object(&object) {1098- Ok(frame) => frame,1099- Err(error) => {1100- return Ok(Some(MoqMediaReadOutcome::Malformed(format!(1101- "Invalid upstream MoQ media object in group {} object {}: {error}",1102- cursor.sequence, cursor.object_count1103- ))));1104- }1105- };1106-1107- let first_object = cursor.object_count == 0;1108- let frame_contiguous = match validate_moq_group_frame(1109- cursor.sequence,1110- first_object,1111- self.last_frame_sequence,1112- &frame,1113- ) {1114- Ok(frame_contiguous) => frame_contiguous,1115- Err(_error) if first_object && cursor.replacement_for_cancelled_group => {1116- self.last_group_sequence = Some(cursor.sequence);1117- self.current_group = None;1118- self.waiting_for_keyframe = true;1119- return Ok(Some(MoqMediaReadOutcome::Dropped {1120- reason: KeyframeRequestReasonV3::TransportGap,1121- }));1122- }1123- Err(_error) if !first_object => {1124- self.last_group_sequence = Some(cursor.sequence);1125- self.current_group = None;1126- self.waiting_for_keyframe = true;1127- return Ok(Some(MoqMediaReadOutcome::Dropped {1128- reason: KeyframeRequestReasonV3::TransportGap,1129- }));1130- }1131- Err(error) => return Ok(Some(MoqMediaReadOutcome::Malformed(error))),1132- };1133- let discontinuity = self.waiting_for_keyframe1134- || (first_object && cursor.group_gap)1135- || (first_object && !frame_contiguous)1136- || frame.header.flags.contains(FrameFlags::DISCONTINUITY);1137- cursor.object_count += 1;1138- cursor.object_bytes = next_group_bytes;1139- self.last_frame_sequence = Some(frame.header.sequence);1140- self.waiting_for_keyframe = false;1141- return Ok(Some(MoqMediaReadOutcome::Frame {1142- frame,1143- discontinuity,1144- }));1145- }1146- }1147-}1148-1149-fn classify_moq_group_sequence(previous: Option<u64>, current: u64) -> Result<bool, String> {1150- let Some(previous) = previous else {1151- return Ok(false);1152- };1153- if current <= previous {1154- return Err(format!(1155- "Upstream MoQ group sequence did not increase: previous={previous}, current={current}"1156- ));1157- }1158- Ok(previous.checked_add(1) != Some(current))1159-}1160-1161-fn validate_moq_object_bounds(1162- group_sequence: u64,1163- object_count: usize,1164- object_bytes: usize,1165- object_len: usize,1166-) -> Result<usize, String> {1167- let max_objects = MAX_MEDIA_OBJECT_ID_V3 as usize + 1;1168- if object_count >= max_objects {1169- return Err(format!(1170- "Upstream MoQ group {group_sequence} exceeded {max_objects} media objects"1171- ));1172- }1173- let next_group_bytes = object_bytes1174- .checked_add(object_len)1175- .ok_or_else(|| "Upstream MoQ group byte count overflowed".to_string())?;1176- if next_group_bytes > MAX_MEDIA_GROUP_BYTES_V3 {1177- return Err(format!(1178- "Upstream MoQ group {group_sequence} exceeded the {MAX_MEDIA_GROUP_BYTES_V3} byte limit"1179- ));1180- }1181- Ok(next_group_bytes)1182-}1183-1184-fn validate_moq_group_frame(1185- group_sequence: u64,1186- first_object: bool,1187- last_frame_sequence: Option<u64>,1188- frame: &MediaFrame,1189-) -> Result<bool, String> {1190- if first_object1191- && !(frame.header.codec == MediaCodec::H2641192- && frame.header.flags.contains(FrameFlags::KEYFRAME)1193- && frame.header.flags.contains(FrameFlags::CODEC_CONFIG))1194- {1195- return Err(format!(1196- "Upstream MoQ group {group_sequence} did not begin with a configured H.264 keyframe"1197- ));1198- }1199- let contiguous = last_frame_sequence1200- .is_none_or(|previous| previous.checked_add(1) == Some(frame.header.sequence));1201- if !first_object && !contiguous {1202- return Err(format!(1203- "Upstream MoQ group {group_sequence} contains a non-contiguous access-unit sequence"1204- ));1205- }1206- Ok(contiguous)1207-}1208-1209-fn moq_group_error_is_recoverable(error: &moq_net::Error) -> bool {1210- matches!(1211- error,1212- moq_net::Error::Cancel1213- | moq_net::Error::Old1214- | moq_net::Error::Timeout1215- | moq_net::Error::Dropped1216- | moq_net::Error::CacheFull1217- | moq_net::Error::Remote(0 | 2 | 3 | 24 | 26)1218- )1219-}1220-1221-fn moq_group_error_reason(error: &moq_net::Error) -> KeyframeRequestReasonV3 {1222- if matches!(error, moq_net::Error::Timeout | moq_net::Error::Remote(3)) {1223- KeyframeRequestReasonV3::DeliveryTimeout1224- } else {1225- KeyframeRequestReasonV3::TransportGap1226- }1227-}1228-1229-1230-enum MediaObjectReadOutcome {1231- Frame {1232- object_index: u64,1233- frame: MediaFrame,1234- },1235- Dropped {1236- object_index: u64,1237- },1238- Malformed(String),1239-}1240-1241-impl MediaObjectReadOutcome {1242- fn object_index(&self) -> Option<u64> {1243- match self {1244- Self::Frame { object_index, .. } | Self::Dropped { object_index } => {1245- Some(*object_index)1246- }1247- Self::Malformed(_) => None,1248- }1249- }1250-1251- fn is_fast_forward_barrier(&self) -> bool {1252- let Self::Frame { frame, .. } = self else {1253- return false;1254- };1255- frame.header.flags.contains(FrameFlags::KEYFRAME)1256- && frame.header.flags.contains(FrameFlags::CODEC_CONFIG)1257- && frame.header.flags.contains(FrameFlags::DISCONTINUITY)1258- }1259-}1260-1261-1262-struct MediaObjectReorder {1263- next_object_index: u64,1264- completed: BTreeMap<u64, MediaObjectReadOutcome>,1265-}1266-1267-impl MediaObjectReorder {1268- fn new(first_object_index: u64) -> Self {1269- Self {1270- next_object_index: first_object_index,1271- completed: BTreeMap::new(),1272- }1273- }1274-1275- fn pending_len(&self) -> usize {1276- self.completed.len()1277- }1278-1279- fn push(1280- &mut self,1281- outcome: MediaObjectReadOutcome,1282- ) -> Result<Option<MediaObjectReadOutcome>, String> {1283- let Some(object_index) = outcome.object_index() else {1284- // Malformed objects remain terminal as soon as their read completes.1285- return Ok(Some(outcome));1286- };1287- if object_index < self.next_object_index {1288- return Ok(Some(outcome));1289- }1290- if outcome.is_fast_forward_barrier() {1291- self.completed1292- .retain(|completed_index, _| *completed_index > object_index);1293- self.next_object_index = object_index1294- .checked_add(1)1295- .ok_or_else(|| "Media object reorder index overflowed".to_string())?;1296- return Ok(Some(outcome));1297- }1298- if self.completed.insert(object_index, outcome).is_some() {1299- return Err(format!(1300- "Media object {object_index} completed more than once"1301- ));1302- }1303- self.take_next()1304- }1305-1306- fn take_next(&mut self) -> Result<Option<MediaObjectReadOutcome>, String> {1307- let Some(outcome) = self.completed.remove(&self.next_object_index) else {1308- return Ok(None);1309- };1310- self.next_object_index = self1311- .next_object_index1312- .checked_add(1)1313- .ok_or_else(|| "Media object reorder index overflowed".to_string())?;1314- Ok(Some(outcome))1315- }1316-}1317-1318-struct MediaObjectReceiver {1319- connection: iroh::endpoint::Connection,1320- reads: tokio::task::JoinSet<MediaObjectReadOutcome>,1321- reorder: MediaObjectReorder,1322- next_object_index: u64,1323- connection_closed: bool,1324-}1325-1326-impl MediaObjectReceiver {1327- fn new(connection: iroh::endpoint::Connection) -> Self {1328- Self {1329- connection,1330- reads: tokio::task::JoinSet::new(),1331- reorder: MediaObjectReorder::new(1),1332- next_object_index: 0,1333- connection_closed: false,1334- }1335- }1336-1337- async fn next(&mut self) -> Result<Option<MediaObjectReadOutcome>, String> {1338- loop {1339- if let Some(completed) = self.reorder.take_next()? {1340- return Ok(Some(completed));1341- }1342- if self.connection_closed && self.reads.is_empty() {1343- if self.reorder.pending_len() != 0 {1344- return Err("Media connection closed with an incomplete object order".into());1345- }1346- return Ok(None);1347- }1348-1349- tokio::select! {1350- biased;1351- completed = self.reads.join_next(), if !self.reads.is_empty() => {1352- let completed = completed1353- .ok_or_else(|| "Media object reader ended unexpectedly".to_string())?1354- .map_err(|error| format!("Media object reader task failed: {error}"))?;1355- if let Some(completed) = self.reorder.push(completed)? {1356- return Ok(Some(completed));1357- }1358- }1359- accepted = self.connection.accept_uni(), if !self.connection_closed1360- && self.reads.len() + self.reorder.pending_len()1361- < CLIENT_MEDIA_OBJECT_CAPACITY => {1362- let mut stream = match accepted {1363- Ok(stream) => stream,1364- Err(_) => {1365- self.connection_closed = true;1366- continue;1367- }1368- };1369- self.next_object_index = self.next_object_index.checked_add(1)1370- .ok_or_else(|| "Media object index overflowed".to_string())?;1371- let object_index = self.next_object_index;1372- self.reads.spawn(async move {1373- match tokio::time::timeout(1374- CLIENT_MEDIA_OBJECT_READ_TIMEOUT,1375- read_media_object(&mut stream),1376- )1377- .await1378- {1379- Err(_) => MediaObjectReadOutcome::Dropped { object_index },1380- Ok(Err(ProtocolError::Io(_))) => {1381- MediaObjectReadOutcome::Dropped { object_index }1382- }1383- Ok(Err(error)) => {1384- MediaObjectReadOutcome::Malformed(format!(1385- "Invalid media object: {error}"1386- ))1387- }1388- Ok(Ok(frame)) => MediaObjectReadOutcome::Frame {1389- object_index,1390- frame,1391- },1392- }1393- });1394- }1395- }1396- }1397- }1398-}1399-1400-impl Drop for MediaObjectReceiver {1401- fn drop(&mut self) {1402- self.reads.abort_all();1403- }1404-}1405-1406-1407-enum MediaObjectReadOutcomeV3 {1408- Object {1409- accept_index: u64,1410- object: MediaObjectV3,1411- },1412- Dropped {1413- accept_index: u64,1414- reason: KeyframeRequestReasonV3,1415- },1416- Malformed(String),1417-}1418-1419-impl MediaObjectReadOutcomeV3 {1420- fn accept_index(&self) -> Option<u64> {1421- match self {1422- Self::Object { accept_index, .. } | Self::Dropped { accept_index, .. } => {1423- Some(*accept_index)1424- }1425- Self::Malformed(_) => None,1426- }1427- }1428-1429- fn is_fast_forward_barrier(&self) -> bool {1430- let Self::Object { object, .. } = self else {1431- return false;1432- };1433- object.header.object_id == 01434- && object.header.flags.contains(FrameFlags::KEYFRAME)1435- && object.header.flags.contains(FrameFlags::CODEC_CONFIG)1436- && object.header.flags.contains(FrameFlags::DISCONTINUITY)1437- }1438-}1439-1440-1441-struct MediaObjectReorderV3 {1442- next_accept_index: u64,1443- completed: BTreeMap<u64, MediaObjectReadOutcomeV3>,1444-}1445-1446-impl MediaObjectReorderV3 {1447- fn new(first_accept_index: u64) -> Self {1448- Self {1449- next_accept_index: first_accept_index,1450- completed: BTreeMap::new(),1451- }1452- }1453-1454- fn pending_len(&self) -> usize {1455- self.completed.len()1456- }1457-1458- fn push(1459- &mut self,1460- outcome: MediaObjectReadOutcomeV3,1461- ) -> Result<Option<MediaObjectReadOutcomeV3>, String> {1462- let Some(accept_index) = outcome.accept_index() else {1463- return Ok(Some(outcome));1464- };1465- if accept_index < self.next_accept_index {1466- // A discontinuity barrier may advance beyond older in-flight1467- // reads. Their eventual timeout/reset outcomes belong to the1468- // superseded GOP and must not poison the recovered sequence.1469- return Ok(None);1470- }1471- if self.completed.insert(accept_index, outcome).is_some() {1472- return Err(format!(1473- "Duplicate media v3 accept index {accept_index} completed"1474- ));1475- }1476- if self1477- .completed1478- .get(&accept_index)1479- .is_some_and(MediaObjectReadOutcomeV3::is_fast_forward_barrier)1480- {1481- self.completed.retain(|index, _| *index >= accept_index);1482- self.next_accept_index = accept_index;1483- }1484- self.take_next()1485- }1486-1487- fn take_next(&mut self) -> Result<Option<MediaObjectReadOutcomeV3>, String> {1488- let Some(outcome) = self.completed.remove(&self.next_accept_index) else {1489- return Ok(None);1490- };1491- self.next_accept_index = self1492- .next_accept_index1493- .checked_add(1)1494- .ok_or_else(|| "Media v3 accept index overflowed".to_string())?;1495- Ok(Some(outcome))1496- }1497-}1498-1499-struct MediaObjectReceiverV3 {1500- connection: iroh::endpoint::Connection,1501- reads: tokio::task::JoinSet<MediaObjectReadOutcomeV3>,1502- reorder: MediaObjectReorderV3,1503- next_accept_index: u64,1504- connection_closed: bool,1505-}1506-1507-impl MediaObjectReceiverV3 {1508- fn new(connection: iroh::endpoint::Connection) -> Self {1509- Self {1510- connection,1511- reads: tokio::task::JoinSet::new(),1512- reorder: MediaObjectReorderV3::new(1),1513- next_accept_index: 0,1514- connection_closed: false,1515- }1516- }1517-1518- async fn next(&mut self) -> Result<Option<MediaObjectReadOutcomeV3>, String> {1519- loop {1520- if let Some(completed) = self.reorder.take_next()? {1521- return Ok(Some(completed));1522- }1523- if self.connection_closed && self.reads.is_empty() {1524- if self.reorder.pending_len() != 0 {1525- return Err("Media v3 connection closed with incomplete object order".into());1526- }1527- return Ok(None);1528- }1529-1530- tokio::select! {1531- biased;1532- completed = self.reads.join_next(), if !self.reads.is_empty() => {1533- let completed = completed1534- .ok_or_else(|| "Media v3 object reader ended unexpectedly".to_string())?1535- .map_err(|error| format!("Media v3 object reader task failed: {error}"))?;1536- if let Some(completed) = self.reorder.push(completed)? {1537- return Ok(Some(completed));1538- }1539- }1540- accepted = self.connection.accept_uni(), if !self.connection_closed1541- && self.reads.len() + self.reorder.pending_len()1542- < CLIENT_MEDIA_OBJECT_CAPACITY => {1543- let mut stream = match accepted {1544- Ok(stream) => stream,1545- Err(_) => {1546- self.connection_closed = true;1547- continue;1548- }1549- };1550- self.next_accept_index = self.next_accept_index.checked_add(1)1551- .ok_or_else(|| "Media v3 accept index overflowed".to_string())?;1552- let accept_index = self.next_accept_index;1553- self.reads.spawn(async move {1554- match tokio::time::timeout(1555- CLIENT_MEDIA_OBJECT_READ_TIMEOUT,1556- read_media_object_v3(&mut stream),1557- )1558- .await1559- {1560- Err(_) => MediaObjectReadOutcomeV3::Dropped {1561- accept_index,1562- reason: KeyframeRequestReasonV3::DeliveryTimeout,1563- },1564- Ok(Err(ProtocolError::Io(_))) => MediaObjectReadOutcomeV3::Dropped {1565- accept_index,1566- reason: KeyframeRequestReasonV3::TransportGap,1567- },1568- Ok(Err(error)) => MediaObjectReadOutcomeV3::Malformed(format!(1569- "Invalid media v3 object: {error}"1570- )),1571- Ok(Ok(object)) => MediaObjectReadOutcomeV3::Object {1572- accept_index,1573- object,1574- },1575- }1576- });1577- }1578- }1579- }1580- }1581-}1582-1583-impl Drop for MediaObjectReceiverV3 {1584- fn drop(&mut self) {1585- self.reads.abort_all();1586- }1587-}1588-1589-1590-enum MediaObjectSequenceDecision {1591- Deliver { discontinuity: bool },1592- DropLate,1593- DropUntilKeyframe,1594-}1595-1596-1597-struct MediaObjectSequence {1598- last_sequence: Option<u64>,1599- last_object_index: u64,1600- waiting_for_keyframe: bool,1601-}1602-1603-impl MediaObjectSequence {1604- fn new() -> Self {1605- Self {1606- waiting_for_keyframe: true,1607- ..Self::default()1608- }1609- }1610-1611- fn note_dropped_object(&mut self, object_index: u64) -> bool {1612- if object_index <= self.last_object_index {1613- return false;1614- }1615- self.waiting_for_keyframe = true;1616- true1617- }1618-1619- fn classify(&mut self, object_index: u64, frame: &MediaFrame) -> MediaObjectSequenceDecision {1620- if object_index <= self.last_object_index1621- || self1622- .last_sequence1623- .is_some_and(|sequence| frame.header.sequence <= sequence)1624- {1625- return MediaObjectSequenceDecision::DropLate;1626- }1627-1628- let keyframe = frame.header.flags.contains(FrameFlags::KEYFRAME)1629- && frame.header.flags.contains(FrameFlags::CODEC_CONFIG);1630- let sequence_contiguous = self1631- .last_sequence1632- .is_none_or(|sequence| sequence.checked_add(1) == Some(frame.header.sequence));1633- if !keyframe && (self.waiting_for_keyframe || !sequence_contiguous) {1634- self.waiting_for_keyframe = true;1635- return MediaObjectSequenceDecision::DropUntilKeyframe;1636- }1637-1638- let discontinuity = frame.header.flags.contains(FrameFlags::DISCONTINUITY)1639- || self.waiting_for_keyframe1640- || !sequence_contiguous;1641- self.last_sequence = Some(frame.header.sequence);1642- self.last_object_index = object_index;1643- self.waiting_for_keyframe = false;1644- MediaObjectSequenceDecision::Deliver { discontinuity }1645- }1646-}1647-1648-1649-enum MediaObjectSequenceDecisionV3 {1650- Deliver { discontinuity: bool },1651- DropLate,1652- DropUntilKeyframe,1653-}1654-1655-1656-struct MediaObjectSequenceV3 {1657- group_id: Option<u64>,1658- last_object_id: Option<u32>,1659- last_sequence: Option<u64>,1660- group_payload_bytes: usize,1661- waiting_for_keyframe: bool,1662-}1663-1664-impl MediaObjectSequenceV3 {1665- fn new() -> Self {1666- Self {1667- waiting_for_keyframe: true,1668- ..Self::default()1669- }1670- }1671-1672- fn note_dropped_object(&mut self) -> bool {1673- let entered = !self.waiting_for_keyframe;1674- self.waiting_for_keyframe = true;1675- entered1676- }1677-1678- fn classify(&mut self, object: &MediaObjectV3) -> MediaObjectSequenceDecisionV3 {1679- let header = &object.header;1680- if self1681- .group_id1682- .is_some_and(|group_id| header.group_id < group_id)1683- || self1684- .last_sequence1685- .is_some_and(|sequence| header.sequence <= sequence)1686- {1687- return MediaObjectSequenceDecisionV3::DropLate;1688- }1689-1690- let new_group = self.group_id != Some(header.group_id);1691- let recovery_keyframe = header.object_id == 01692- && header.flags.contains(FrameFlags::KEYFRAME)1693- && header.flags.contains(FrameFlags::CODEC_CONFIG);1694- if new_group && !recovery_keyframe {1695- self.waiting_for_keyframe = true;1696- return MediaObjectSequenceDecisionV3::DropUntilKeyframe;1697- }1698- if !new_group && self.waiting_for_keyframe {1699- return MediaObjectSequenceDecisionV3::DropUntilKeyframe;1700- }1701-1702- let sequence_contiguous = self1703- .last_sequence1704- .is_none_or(|sequence| sequence.checked_add(1) == Some(header.sequence));1705- let object_contiguous = new_group1706- || self1707- .last_object_id1708- .is_some_and(|object_id| object_id.checked_add(1) == Some(header.object_id));1709- let next_group_bytes = if new_group {1710- object.payload.len()1711- } else {1712- self.group_payload_bytes1713- .saturating_add(object.payload.len())1714- };1715- if (!sequence_contiguous && !new_group)1716- || !object_contiguous1717- || next_group_bytes > MAX_MEDIA_GROUP_BYTES_V31718- {1719- self.waiting_for_keyframe = true;1720- return MediaObjectSequenceDecisionV3::DropUntilKeyframe;1721- }1722-1723- let discontinuity = header.flags.contains(FrameFlags::DISCONTINUITY)1724- || self.waiting_for_keyframe1725- || !sequence_contiguous;1726- self.group_id = Some(header.group_id);1727- self.last_object_id = Some(header.object_id);1728- self.last_sequence = Some(header.sequence);1729- self.group_payload_bytes = next_group_bytes;1730- self.waiting_for_keyframe = false;1731- MediaObjectSequenceDecisionV3::Deliver { discontinuity }1732- }1733-}1734-1735-1736-struct InputAvailability {1737- relative_pointer: bool,1738- pointer_position_feedback: bool,1739- absolute_pointer: bool,1740- keyboard: bool,1741- text: bool,1742- gamepad: bool,1743- input_ack: bool,1744- control: bool,1745-}1746-1747-impl InputAvailability {1748- fn from_capabilities(capabilities: &[Capability]) -> Self {1749- let relative_pointer = capabilities.contains(&Capability::RelativePointer);1750- let pointer_position_feedback = capabilities.contains(&Capability::PointerPositionFeedback);1751- let absolute_pointer = capabilities.contains(&Capability::AbsolutePointer);1752- let keyboard = capabilities.contains(&Capability::Keyboard);1753- let text = capabilities.contains(&Capability::Text);1754- let gamepad = capabilities.contains(&Capability::Gamepad);1755- let input_ack = capabilities.contains(&Capability::InputAck);1756- Self {1757- relative_pointer,1758- pointer_position_feedback,1759- absolute_pointer,1760- keyboard,1761- text,1762- gamepad,1763- input_ack,1764- control: relative_pointer || absolute_pointer || keyboard || text || gamepad,1765- }1766- }1767-}1768-1769-1770-1771-pub enum PointerFeedbackPayload {1772- Position {1773- sequence: u64,1774- position: Option<PointerPosition>,1775- 1776- pointer_visible: Option<bool>,1777- },1778- Terminal {1779- reason: PointerFeedbackTerminalReason,1780- },1781-}1782-1783-1784-1785-pub enum PointerFeedbackTerminalReason {1786- Eof,1787- Malformed,1788-}1789-1790-1791-struct RelativePointerAccumulator {1792- dx: i64,1793- dy: i64,1794-}1795-1796-impl RelativePointerAccumulator {1797- fn push(&mut self, dx: i32, dy: i32) {1798- self.dx = self.dx.saturating_add(i64::from(dx));1799- self.dy = self.dy.saturating_add(i64::from(dy));1800- }1801-1802- fn take(&mut self) -> Option<InputEvent> {1803- if self.dx == 0 && self.dy == 0 {1804- return None;1805- }1806- let dx = self.dx.clamp(1807- i64::from(RELATIVE_POINTER_DELTA_MIN),1808- i64::from(RELATIVE_POINTER_DELTA_MAX),1809- ) as i32;1810- let dy = self.dy.clamp(1811- i64::from(RELATIVE_POINTER_DELTA_MIN),1812- i64::from(RELATIVE_POINTER_DELTA_MAX),1813- ) as i32;1814- let event = InputEvent::MouseMoveRelative { dx, dy };1815- self.dx -= i64::from(dx);1816- self.dy -= i64::from(dy);1817- Some(event)1818- }1819-1820- fn is_pending(&self) -> bool {1821- self.dx != 0 || self.dy != 01822- }1823-}1824-1825-fn stage_relative_input(1826- pending: &mut RelativePointerAccumulator,1827- event: InputEvent,1828-) -> Option<InputEvent> {1829- match event {1830- InputEvent::MouseMoveRelative { dx, dy } => {1831- pending.push(dx, dy);1832- None1833- }1834- event => Some(event),1835- }1836-}1837-1838-async fn negotiate_v1(1839- send: &mut iroh::endpoint::SendStream,1840- recv: &mut iroh::endpoint::RecvStream,1841- nonce: [u8; 16],1842- capabilities: Vec<Capability>,1843- required: Option<Capability>,1844- stream_name: &str,1845- invitation: Option<&str>,1846-) -> Result<NegotiatedV1Stream, String> {1847- let mut hello = ClientHello::new("portal/0.1.0", nonce, capabilities.clone());1848- if let Some(invitation) = invitation {1849- hello = hello.with_invitation(invitation);1850- }1851- write_client_hello(send, &hello)1852- .await1853- .map_err(|e| format!("Failed to send {stream_name} handshake: {e}"))?;1854- let response = tokio::time::timeout(Duration::from_secs(10), read_host_hello(recv))1855- .await1856- .map_err(|_| format!("Timed out waiting for {stream_name} handshake"))?1857- .map_err(|e| format!("Invalid {stream_name} handshake: {e}"))?1858- .ok_or_else(|| format!("Host closed during {stream_name} handshake"))?;1859- if !response.accepted {1860- return Err(format!(1861- "Host rejected {stream_name} stream: {}",1862- response.message.as_deref().unwrap_or("unspecified reason")1863- ));1864- }1865- if let Some(required) = required1866- && !response.capabilities.contains(&required)1867- {1868- return Err(format!(1869- "Host accepted {stream_name} without required capability {required:?}"1870- ));1871- }1872- if let Some(unoffered) = response1873- .capabilities1874- .iter()1875- .find(|capability| !capabilities.contains(capability))1876- {1877- return Err(format!(1878- "Host accepted unoffered {stream_name} capability {unoffered:?}"1879- ));1880- }1881- let session_id = response1882- .session_id1883- .ok_or_else(|| format!("Host omitted {stream_name} session ID"))?;1884- Ok(NegotiatedV1Stream {1885- session_id,1886- capabilities: response.capabilities,1887- pointer_surface_dimensions: response.pointer_surface_dimensions,1888- })1889-}1890-1891-async fn open_negotiated_input_stream(1892- endpoint: &Endpoint,1893- address: &iroh::EndpointAddr,1894- nonce: [u8; 16],1895- capabilities: Vec<Capability>,1896-) -> Result<1897- (1898- iroh::endpoint::Connection,1899- iroh::endpoint::SendStream,1900- iroh::endpoint::RecvStream,1901- NegotiatedV1Stream,1902- ),1903- String,1904-> {1905- let connection = endpoint1906- .connect(address.clone(), INPUT_ALPN_V1)1907- .await1908- .map_err(|error| format!("Failed to connect input stream: {error}"))?;1909- let (mut send, mut recv) = connection1910- .open_bi()1911- .await1912- .map_err(|error| format!("Failed to open input stream: {error}"))?;1913- let negotiation = negotiate_v1(1914- &mut send,1915- &mut recv,1916- nonce,1917- capabilities,1918- None,1919- "input",1920- None,1921- )1922- .await?;1923- Ok((connection, send, recv, negotiation))1924-}1925-1926-fn connection_error_is_unsupported_alpn(error: &iroh::endpoint::ConnectionError) -> bool {1927- matches!(1928- error,1929- iroh::endpoint::ConnectionError::ConnectionClosed(close)1930- if close.error_code == iroh::endpoint::TransportErrorCode::crypto(0x78)1931- )1932-}1933-1934-fn connect_error_is_unsupported_alpn(error: &iroh::endpoint::ConnectError) -> bool {1935- match error {1936- iroh::endpoint::ConnectError::Connecting {1937- source: iroh::endpoint::ConnectingError::ConnectionError { source, .. },1938- ..1939- }1940- | iroh::endpoint::ConnectError::Connection { source, .. } => {1941- connection_error_is_unsupported_alpn(source)1942- }1943- _ => false,1944- }1945-}1946-1947-async fn open_negotiated_feedback_stream(1948- endpoint: &Endpoint,1949- address: &iroh::EndpointAddr,1950- nonce: [u8; 16],1951- media_session_id: u64,1952-) -> Result<1953- Option<(1954- iroh::endpoint::Connection,1955- iroh::endpoint::SendStream,1956- iroh::endpoint::RecvStream,1957- )>,1958- String,1959-> {1960- let connection = match endpoint1961- .connect(address.clone(), MEDIA_FEEDBACK_ALPN_V1)1962- .await1963- {1964- Ok(connection) => connection,1965- Err(error) if connect_error_is_unsupported_alpn(&error) => return Ok(None),1966- Err(error) => {1967- return Err(format!(1968- "Failed to connect adaptive feedback stream: {error}"1969- ));1970- }1971- };1972- let (mut send, mut recv) = connection1973- .open_bi()1974- .await1975- .map_err(|error| format!("Failed to open adaptive feedback handshake: {error}"))?;1976- let negotiation = negotiate_v1(1977- &mut send,1978- &mut recv,1979- nonce,1980- vec![Capability::VideoH264],1981- Some(Capability::VideoH264),1982- "adaptive feedback",1983- None,1984- )1985- .await?;1986- if negotiation.session_id != media_session_id {1987- return Err("Host returned mismatched media and adaptive feedback sessions".to_string());1988- }1989- Ok(Some((connection, send, recv)))1990-}1991-1992-fn adaptive_bitrate_state_name(state: AdaptiveBitrateStateV1) -> &'static str {1993- match state {1994- AdaptiveBitrateStateV1::Hold => "hold",1995- AdaptiveBitrateStateV1::Decrease => "decrease",1996- AdaptiveBitrateStateV1::Increase => "increase",1997- }1998-}1999-2000-fn adaptive_bitrate_reason_names(reasons: AdaptiveBitrateReasonFlagsV1) -> Vec<&'static str> {2001- [2002- (AdaptiveBitrateReasonFlagsV1::RTT_INFLATION, "rtt-inflation"),2003- (2004- AdaptiveBitrateReasonFlagsV1::LOSS_OR_CANCELLATION,2005- "loss-or-cancellation",2006- ),2007- (2008- AdaptiveBitrateReasonFlagsV1::SENDER_BACKPRESSURE,2009- "sender-backpressure",2010- ),2011- (2012- AdaptiveBitrateReasonFlagsV1::RECEIVER_QUEUE,2013- "receiver-queue",2014- ),2015- (2016- AdaptiveBitrateReasonFlagsV1::DECODE_BACKLOG,2017- "decode-backlog",2018- ),2019- (2020- AdaptiveBitrateReasonFlagsV1::DELIVERY_LATENCY,2021- "delivery-latency",2022- ),2023- (2024- AdaptiveBitrateReasonFlagsV1::CLEAN_RECOVERY,2025- "clean-recovery",2026- ),2027- (2028- AdaptiveBitrateReasonFlagsV1::FEEDBACK_STALE,2029- "feedback-stale",2030- ),2031- ]2032- .into_iter()2033- .filter_map(|(flag, name)| reasons.contains(flag).then_some(name))2034- .collect()2035-}2036-2037-fn adaptive_bitrate_decision_diagnostic(2038- decision: AdaptiveBitrateDecisionV1,2039-) -> AdaptiveBitrateDecisionDiagnostic {2040- AdaptiveBitrateDecisionDiagnostic {2041- decision_id: decision.decision_id,2042- report_id: decision.report_id,2043- target_kbps: decision.target_kbps,2044- floor_kbps: decision.floor_kbps,2045- ceiling_kbps: decision.ceiling_kbps,2046- state: adaptive_bitrate_state_name(decision.state),2047- reasons: adaptive_bitrate_reason_names(decision.reasons),2048- applied: decision.applied,2049- }2050-}2051-2052-2053-struct AdaptiveDecisionSequence {2054- last_decision_id: Option<u64>,2055- last_report_id: Option<u64>,2056-}2057-2058-impl AdaptiveDecisionSequence {2059- fn accept(&mut self, decision: &AdaptiveBitrateDecisionV1) -> Result<(), String> {2060- if self2061- .last_decision_id2062- .is_some_and(|previous| decision.decision_id <= previous)2063- {2064- return Err(format!(2065- "adaptive decision ID did not increase: previous={:?}, current={}",2066- self.last_decision_id, decision.decision_id2067- ));2068- }2069- if self2070- .last_report_id2071- .is_some_and(|previous| decision.report_id <= previous)2072- {2073- return Err(format!(2074- "adaptive decision report ID did not increase: previous={:?}, current={}",2075- self.last_report_id, decision.report_id2076- ));2077- }2078- self.last_decision_id = Some(decision.decision_id);2079- self.last_report_id = Some(decision.report_id);2080- Ok(())2081- }2082-}2083-2084-async fn emit_paced_adaptive_decisions<F>(2085- generation: u64,2086- mut decisions: tokio::sync::watch::Receiver<Option<AdaptiveBitrateDecisionV1>>,2087- delivery_interval: Duration,2088- mut emit: F,2089-) -> Result<(), String>2090-where2091- F: FnMut(AdaptiveBitrateDecisionPayload) -> Result<(), String>,2092-{2093- let mut delivery = tokio::time::interval(delivery_interval);2094- delivery.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);2095- loop {2096- decisions2097- .changed()2098- .await2099- .map_err(|_| "adaptive decision receiver closed".to_string())?;2100- // The first tick is immediate. Later ticks pace the webview boundary;2101- // any host burst replaces the watch value while this task waits.2102- delivery.tick().await;2103- let Some(decision) = *decisions.borrow_and_update() else {2104- continue;2105- };2106- emit(AdaptiveBitrateDecisionPayload {2107- generation,2108- decision: adaptive_bitrate_decision_diagnostic(decision),2109- })?;2110- }2111-}2112-2113-async fn run_media_feedback_session(2114- app: AppHandle,2115- generation: u64,2116- mut send: iroh::endpoint::SendStream,2117- mut recv: iroh::endpoint::RecvStream,2118- mut reports: tokio::sync::watch::Receiver<Option<AccumulatedMediaFeedback>>,2119-) {2120- let (decision_tx, decision_rx) = tokio::sync::watch::channel(None);2121- let writer = async {2122- let mut last_written = None;2123- loop {2124- reports2125- .changed()2126- .await2127- .map_err(|_| "adaptive feedback sender closed".to_string())?;2128- let accumulated = *reports.borrow_and_update();2129- let Some(accumulated) = accumulated else {2130- continue;2131- };2132- let report = accumulated.report_since(last_written);2133- tokio::time::timeout(2134- CLIENT_MEDIA_FEEDBACK_IO_TIMEOUT,2135- write_media_feedback_report_v1(&mut send, &report),2136- )2137- .await2138- .map_err(|_| "adaptive feedback write timed out".to_string())?2139- .map_err(|error| format!("adaptive feedback write failed: {error}"))?;2140- last_written = Some(accumulated);2141- }2142- 2143- Ok::<(), String>(())2144- };2145- let reader = async {2146- let mut sequence = AdaptiveDecisionSequence::default();2147- loop {2148- let decision = read_adaptive_bitrate_decision_v1(&mut recv)2149- .await2150- .map_err(|error| format!("adaptive decision read failed: {error}"))?2151- .ok_or_else(|| "adaptive decision stream closed".to_string())?;2152- sequence.accept(&decision)?;2153- decision_tx.send_replace(Some(decision));2154- }2155- 2156- Ok::<(), String>(())2157- };2158- let decision_app = app.clone();2159- let decision_emitter = emit_paced_adaptive_decisions(2160- generation,2161- decision_rx,2162- CLIENT_ADAPTIVE_DECISION_DELIVERY_INTERVAL,2163- move |payload| {2164- decision_app2165- .emit("adaptive-bitrate-decision", payload)2166- .map_err(|error| format!("adaptive decision event delivery failed: {error}"))2167- },2168- );2169- let terminal = tokio::select! {2170- terminal = writer => terminal,2171- terminal = reader => terminal,2172- terminal = decision_emitter => terminal,2173- };2174- if let Err(error) = terminal {2175- eprintln!("[client] {error}");2176- if let Err(emit_error) = app.emit(2177- "adaptive-feedback-state",2178- serde_json::json!({2179- "generation": generation,2180- "available": false,2181- "error": error,2182- }),2183- ) {2184- eprintln!("[client] adaptive feedback terminal event delivery failed: {emit_error}");2185- }2186- }2187- retire_media_feedback_generation(&app, generation).await;2188-}2189-2190-async fn open_legacy_negotiated_media_stream(2191- endpoint: &Endpoint,2192- address: &iroh::EndpointAddr,2193- nonce: [u8; 16],2194- invitation: Option<&str>,2195-) -> Result<2196- (2197- iroh::endpoint::Connection,2198- iroh::endpoint::RecvStream,2199- Option<iroh::endpoint::SendStream>,2200- NegotiatedV1Stream,2201- MediaTransport,2202- ),2203- String,2204-> {2205- match endpoint.connect(address.clone(), MEDIA_ALPN_V3).await {2206- Ok(connection) => {2207- let (mut send, mut recv) = connection2208- .open_bi()2209- .await2210- .map_err(|error| format!("Failed to open media v3 handshake: {error}"))?;2211- let negotiation = negotiate_v1(2212- &mut send,2213- &mut recv,2214- nonce,2215- vec![Capability::VideoH264],2216- Some(Capability::VideoH264),2217- "media v3",2218- invitation,2219- )2220- .await?;2221- Ok((2222- connection,2223- recv,2224- Some(send),2225- negotiation,2226- MediaTransport::GroupedObjectsV3,2227- ))2228- }2229- Err(v3_error) if connect_error_is_unsupported_alpn(&v3_error) => {2230- match endpoint.connect(address.clone(), MEDIA_ALPN_V2).await {2231- Ok(connection) => {2232- let (mut send, mut recv) = connection2233- .open_bi()2234- .await2235- .map_err(|error| format!("Failed to open media v2 handshake: {error}"))?;2236- let negotiation = negotiate_v1(2237- &mut send,2238- &mut recv,2239- nonce,2240- vec![Capability::VideoH264],2241- Some(Capability::VideoH264),2242- "media v2",2243- invitation,2244- )2245- .await?;2246- send.finish()2247- .map_err(|error| format!("Failed to finish media v2 handshake: {error}"))?;2248- Ok((2249- connection,2250- recv,2251- None,2252- negotiation,2253- MediaTransport::IndependentObjectsV2,2254- ))2255- }2256- Err(v2_error) if connect_error_is_unsupported_alpn(&v2_error) => {2257- let connection = endpoint2258- .connect(address.clone(), MEDIA_ALPN_V1)2259- .await2260- .map_err(|v1_error| {2261- format!(2262- "Failed to connect media v3 ({v3_error}); v2 compatibility connection failed ({v2_error}); v1 compatibility connection also failed ({v1_error})"2263- )2264- })?;2265- let (mut send, mut recv) = connection2266- .open_bi()2267- .await2268- .map_err(|error| format!("Failed to open media v1 stream: {error}"))?;2269- let negotiation = negotiate_v1(2270- &mut send,2271- &mut recv,2272- nonce,2273- vec![Capability::VideoH264],2274- Some(Capability::VideoH264),2275- "media v1",2276- invitation,2277- )2278- .await?;2279- send.finish()2280- .map_err(|error| format!("Failed to finish media v1 handshake: {error}"))?;2281- Ok((2282- connection,2283- recv,2284- None,2285- negotiation,2286- MediaTransport::ReliableStreamV1,2287- ))2288- }2289- Err(v2_error) => Err(format!(2290- "Media v2 compatibility connection failed without an explicit unsupported-ALPN signal; refusing an unsafe downgrade to v1: {v2_error}"2291- )),2292- }2293- }2294- Err(v3_error) => Err(format!(2295- "Media v3 connection failed without an explicit unsupported-ALPN signal; refusing an unsafe compatibility downgrade: {v3_error}"2296- )),2297- }2298-}2299-2300-async fn open_negotiated_media_stream(2301- endpoint: &Endpoint,2302- address: &iroh::EndpointAddr,2303- nonce: [u8; 16],2304- invitation: Option<&str>,2305-) -> Result<2306- (2307- iroh::endpoint::Connection,2308- iroh::endpoint::RecvStream,2309- Option<iroh::endpoint::SendStream>,2310- NegotiatedV1Stream,2311- MediaTransport,2312- ),2313- String,2314-> {2315- match endpoint.connect(address.clone(), CONTROL_ALPN_V1).await {2316- Ok(connection) => {2317- let (mut send, mut recv) = connection2318- .open_bi()2319- .await2320- .map_err(|error| format!("Failed to open control handshake: {error}"))?;2321- let negotiation = negotiate_v1(2322- &mut send,2323- &mut recv,2324- nonce,2325- vec![Capability::VideoH264],2326- Some(Capability::VideoH264),2327- "control",2328- invitation,2329- )2330- .await?;2331- // CONTROL owns the authenticated host lease. Keep both the2332- // connection and the client->host send leg alive for keyframe2333- // requests while media uses a separate upstream MoQ session.2334- Ok((2335- connection,2336- recv,2337- Some(send),2338- negotiation,2339- MediaTransport::UpstreamMoq,2340- ))2341- }2342- Err(control_error) if connect_error_is_unsupported_alpn(&control_error) => {2343- open_legacy_negotiated_media_stream(endpoint, address, nonce, invitation).await2344- }2345- Err(control_error) => Err(format!(2346- "Control connection failed without an explicit unsupported-ALPN signal; refusing an unsafe media downgrade: {control_error}"2347- )),2348- }2349-}2350-2351-async fn open_upstream_moq_media(2352- endpoint: &Endpoint,2353- address: &iroh::EndpointAddr,2354- session_id: u64,2355-) -> Result<(MoqMediaReceiver, iroh::endpoint::Connection), String> {2356- let broadcast_name = media_moq_broadcast_name(session_id)2357- .map_err(|error| format!("Invalid MoQ media session name: {error}"))?;2358- let moq = Moq::new(endpoint.clone());2359- let mut session =2360- tokio::time::timeout(CLIENT_MOQ_CONNECT_TIMEOUT, moq.connect(address.clone()))2361- .await2362- .map_err(|_| "Timed out connecting upstream MoQ media session".to_string())?2363- .map_err(|error| format!("Failed to connect upstream MoQ media session: {error:#}"))?;2364- if session.remote_id() != address.id {2365- session.close(1, b"remote identity mismatch");2366- return Err(format!(2367- "Upstream MoQ connected to unexpected peer {}; expected {}",2368- session.remote_id(),2369- address.id2370- ));2371- }2372- let diagnostics_connection = session.conn().clone();2373- let broadcast = tokio::time::timeout(2374- CLIENT_MOQ_SUBSCRIBE_TIMEOUT,2375- session.subscribe(&broadcast_name),2376- )2377- .await2378- .map_err(|_| format!("Timed out waiting for upstream MoQ broadcast {broadcast_name}"))?2379- .map_err(|error| {2380- format!("Failed to subscribe to upstream MoQ broadcast {broadcast_name}: {error}")2381- })?;2382- let catalog = subscribe_goq_video_track(&broadcast, CLIENT_MOQ_SUBSCRIBE_TIMEOUT).await?;2383- eprintln!("[client] moq catalog: {}", catalog.mode.label());2384- Ok((2385- MoqMediaReceiver::new(moq, session, broadcast, catalog.track),2386- diagnostics_connection,2387- ))2388-}2389-2390-async fn run_media_control_writer_v3(2391- mut stream: iroh::endpoint::SendStream,2392- mut requests: tokio::sync::mpsc::Receiver<(KeyframeRequestReasonV3, Option<u64>)>,2393-) {2394- let mut request_id = 0_u64;2395- while let Some((reason, last_sequence)) = requests.recv().await {2396- let Some(next_request_id) = request_id.checked_add(1) else {2397- eprintln!("[client] media v3 keyframe request id overflowed");2398- break;2399- };2400- request_id = next_request_id;2401- let request = MediaControlRequestV3::request_keyframe(request_id, last_sequence, reason);2402- match tokio::time::timeout(2403- Duration::from_secs(1),2404- write_media_control_request_v3(&mut stream, &request),2405- )2406- .await2407- {2408- Ok(Ok(())) => {}2409- Ok(Err(error)) => {2410- eprintln!("[client] media v3 keyframe request failed: {error}");2411- break;2412- }2413- Err(_) => {2414- eprintln!("[client] media v3 keyframe request timed out");2415- break;2416- }2417- }2418- }2419- let _ = stream.finish();2420-}2421-2422-fn parse_keyframe_request_reason(reason: &str) -> Result<KeyframeRequestReasonV3, String> {2423- match reason {2424- "join" => Ok(KeyframeRequestReasonV3::Join),2425- "transport-gap" => Ok(KeyframeRequestReasonV3::TransportGap),2426- "delivery-timeout" => Ok(KeyframeRequestReasonV3::DeliveryTimeout),2427- "discontinuity" | "decoder-reset" | "decoder-error" => {2428- Ok(KeyframeRequestReasonV3::DecoderReset)2429- }2430- "frontend-backpressure" => Ok(KeyframeRequestReasonV3::FrontendBackpressure),2431- _ => Err(format!("Unsupported keyframe request reason: {reason}")),2432- }2433-}2434-2435-fn try_queue_media_keyframe_request(2436- sender: Option<&tokio::sync::mpsc::Sender<(KeyframeRequestReasonV3, Option<u64>)>>,2437- reason: KeyframeRequestReasonV3,2438- last_sequence: Option<u64>,2439-) {2440- if let Some(sender) = sender {2441- let _ = sender.try_send((reason, last_sequence));2442- }2443-}2444-2445-fn input_capability_offers(grants: InvitationGrants) -> Vec<Vec<Capability>> {2446- let base = [2447- vec![2448- Capability::RelativePointer,2449- Capability::PointerPositionFeedback,2450- Capability::PointerVisibilityFeedback,2451- Capability::AbsolutePointer,2452- Capability::Keyboard,2453- Capability::Text,2454- Capability::Gamepad,2455- ],2456- vec![2457- Capability::RelativePointer,2458- Capability::PointerPositionFeedback,2459- Capability::AbsolutePointer,2460- Capability::Keyboard,2461- Capability::Text,2462- Capability::Gamepad,2463- ],2464- vec![2465- Capability::RelativePointer,2466- Capability::AbsolutePointer,2467- Capability::Keyboard,2468- Capability::Text,2469- Capability::Gamepad,2470- ],2471- vec![2472- Capability::AbsolutePointer,2473- Capability::Keyboard,2474- Capability::Text,2475- Capability::Gamepad,2476- ],2477- ];2478- let has_input_grant = grants.contains(InvitationGrants::POINTER_KEYBOARD)2479- || grants.contains(InvitationGrants::GAMEPAD);2480- let mut offers = Vec::with_capacity(base.len() * 2);2481- if has_input_grant {2482- for mut offer in base.clone() {2483- offer.push(Capability::InputAck);2484- offers.push(offer);2485- }2486- }2487- offers.extend(base);2488- for offer in &mut offers {2489- offer.retain(|capability| match capability {2490- Capability::Gamepad => grants.contains(InvitationGrants::GAMEPAD),2491- Capability::AbsolutePointer2492- | Capability::RelativePointer2493- | Capability::Keyboard2494- | Capability::Text2495- | Capability::PointerPositionFeedback2496- | Capability::PointerVisibilityFeedback => {2497- grants.contains(InvitationGrants::POINTER_KEYBOARD)2498- }2499- Capability::InputAck => has_input_grant,2500- _ => true,2501- });2502- }2503- offers.dedup();2504- offers2505-}2506-2507-fn input_event_allowed(capabilities: &[Capability], event: &InputEvent) -> bool {2508- match event {2509- InputEvent::Probe => capabilities.contains(&Capability::InputAck),2510- InputEvent::MouseMove { .. } => capabilities.contains(&Capability::AbsolutePointer),2511- InputEvent::MouseMoveRelative { .. } => capabilities.contains(&Capability::RelativePointer),2512- InputEvent::MousePositionSync { .. } => capabilities.contains(&Capability::RelativePointer),2513- InputEvent::MouseClick { .. }2514- | InputEvent::MouseDown { .. }2515- | InputEvent::MouseUp { .. }2516- | InputEvent::MouseScroll { .. } => {2517- capabilities.contains(&Capability::RelativePointer)2518- || capabilities.contains(&Capability::AbsolutePointer)2519- }2520- InputEvent::KeyDown { .. } | InputEvent::KeyUp { .. } | InputEvent::KeyClick { .. } => {2521- capabilities.contains(&Capability::Keyboard)2522- }2523- InputEvent::Text { .. } => capabilities.contains(&Capability::Text),2524- InputEvent::Gamepad { .. } => capabilities.contains(&Capability::Gamepad),2525- }2526-}2527-2528-async fn write_client_input_event(2529- stream: &mut iroh::endpoint::SendStream,2530- event: &InputEvent,2531- use_v1: bool,2532- diagnostics: Option<&Arc<StdMutex<NetworkSessionDiagnostics>>>,2533-) -> Result<(), String> {2534- if let Some(diagnostics) = diagnostics {2535- lock_network_diagnostics(diagnostics).begin_input_send(Instant::now());2536- }2537- if use_v1 {2538- write_input_event(stream, event)2539- .await2540- .map_err(|error| error.to_string())2541- } else {2542- let json = serde_json::to_string(event).map_err(|error| error.to_string())?;2543- stream2544- .write_all(format!("{json}\n").as_bytes())2545- .await2546- .map_err(|error| error.to_string())2547- }2548-}2549-2550-fn observe_input_ack_if_negotiated(2551- diagnostics: &StdMutex<NetworkSessionDiagnostics>,2552- negotiated: bool,2553- sequence: u64,2554- now: Instant,2555-) -> Result<(), String> {2556- if !negotiated {2557- return Ok(());2558- }2559- lock_network_diagnostics(diagnostics).observe_input_ack(sequence, now)2560-}2561-2562-2563-struct AudioReorderBuffer {2564- expected_sequence: Option<u64>,2565- packets: BTreeMap<u64, AudioPacket>,2566-}2567-2568-2569-struct OrderedAudioPacket {2570- packet: AudioPacket,2571- discontinuity: bool,2572-}2573-2574-impl AudioReorderBuffer {2575- const CAPACITY: usize = 3;2576-2577- fn insert(&mut self, packet: AudioPacket) -> Result<(Vec<OrderedAudioPacket>, u64), String> {2578- let sequence = packet.header.sequence;2579- let expected = self.expected_sequence.get_or_insert(sequence);2580- if sequence < *expected || self.packets.contains_key(&sequence) {2581- return Ok((Vec::new(), 0));2582- }2583- self.packets.insert(sequence, packet);2584-2585- let mut dropped = 0_u64;2586- let mut discontinuity = false;2587- if !self.packets.contains_key(expected) && self.packets.len() >= Self::CAPACITY {2588- let next = *self2589- .packets2590- .first_key_value()2591- .expect("capacity check guarantees one packet")2592- .0;2593- dropped = next.saturating_sub(*expected);2594- *expected = next;2595- discontinuity = true;2596- }2597-2598- let mut ordered = Vec::with_capacity(Self::CAPACITY);2599- while let Some(packet) = self.packets.remove(expected) {2600- ordered.push(OrderedAudioPacket {2601- packet,2602- discontinuity,2603- });2604- discontinuity = false;2605- *expected = expected2606- .checked_add(1)2607- .ok_or_else(|| "Audio sequence overflowed".to_string())?;2608- }2609- Ok((ordered, dropped))2610- }2611-}2612-2613-fn encode_audio_channel_packet(2614- generation: u64,2615- delivery_id: u64,2616- packet: AudioPacket,2617- force_discontinuity: bool,2618-) -> Result<Vec<u8>, String> {2619- let protocol_packet = if force_discontinuity {2620- let header = AudioPacketHeader::opus(2621- packet.payload.len(),2622- packet.header.sequence,2623- packet.header.capture_timestamp_us,2624- packet.header.pts_us,2625- AudioFlags::DISCONTINUITY,2626- )2627- .map_err(|error| error.to_string())?;2628- AudioPacket::new(header, packet.payload).map_err(|error| error.to_string())?2629- } else {2630- packet2631- };2632- let datagram = protocol_packet2633- .encode_datagram()2634- .map_err(|error| error.to_string())?;2635- let mut envelope = Vec::with_capacity(AUDIO_CHANNEL_HEADER_LEN + datagram.len());2636- envelope.extend_from_slice(&AUDIO_CHANNEL_MAGIC);2637- envelope.extend_from_slice(&AUDIO_CHANNEL_VERSION.to_be_bytes());2638- envelope.extend_from_slice(&(AUDIO_CHANNEL_HEADER_LEN as u16).to_be_bytes());2639- envelope.extend_from_slice(&generation.to_be_bytes());2640- envelope.extend_from_slice(&delivery_id.to_be_bytes());2641- envelope.extend_from_slice(&datagram);2642- debug_assert_eq!(envelope.len(), AUDIO_CHANNEL_HEADER_LEN + datagram.len());2643- Ok(envelope)2644-}2645-2646-struct AudioStartRequest {2647- address: iroh::EndpointAddr,2648- handshake_nonce: [u8; 16],2649- media_session_id: Option<u64>,2650- audio_supported: bool,2651- audio_channel: Channel<Response>,2652- audio_deliveries: Arc<StdMutex<AudioDeliveryState>>,2653- connection_generation: Arc<AtomicU64>,2654- generation: u64,2655-}2656-2657-async fn try_start_audio(2658- app: AppHandle,2659- endpoint: &Endpoint,2660- request: AudioStartRequest,2661-) -> Result<iroh::endpoint::Connection, String> {2662- if !request.audio_supported {2663- return Err("WebCodecs Opus AudioDecoder is unavailable".to_string());2664- }2665- let media_session_id = request2666- .media_session_id2667- .ok_or_else(|| "The connected host protocol does not negotiate audio".to_string())?;2668- let audio_connection = tokio::time::timeout(2669- Duration::from_secs(3),2670- endpoint.connect(request.address, AUDIO_ALPN_V1),2671- )2672- .await2673- .map_err(|_| "Timed out connecting optional audio".to_string())?2674- .map_err(|error| format!("Audio connection unavailable: {error}"))?;2675- let (mut send, mut recv) = audio_connection2676- .open_bi()2677- .await2678- .map_err(|error| format!("Failed to open audio handshake: {error}"))?;2679- let negotiation = negotiate_v1(2680- &mut send,2681- &mut recv,2682- request.handshake_nonce,2683- vec![Capability::AudioOpus],2684- Some(Capability::AudioOpus),2685- "audio",2686- None,2687- )2688- .await?;2689- if negotiation.session_id != media_session_id {2690- audio_connection.close(1_u32.into(), b"audio session mismatch");2691- return Err("Host returned mismatched media and audio sessions".to_string());2692- }2693- send.finish()2694- .map_err(|error| format!("Failed to finish audio handshake: {error}"))?;2695-2696- let task_audio_connection = audio_connection.clone();2697- tokio::spawn(async move {2698- let audio_channel = request.audio_channel;2699- let audio_deliveries = request.audio_deliveries;2700- let connection_generation = request.connection_generation;2701- let generation = request.generation;2702- let mut reorder = AudioReorderBuffer::default();2703- let mut transport_received_total = 0_u64;2704- let mut sequence_dropped_total = 0_u64;2705- let mut frontend_dropped_total = 0_u64;2706- let mut frontend_sent_total = 0_u64;2707- let mut pending_discontinuity = false;2708- let mut last_stats = Instant::now();2709-2710- loop {2711- let datagram = match task_audio_connection.read_datagram().await {2712- Ok(datagram) => datagram,2713- Err(error) => {2714- emit_audio_event_if_current(2715- &app,2716- "audio-state",2717- &connection_generation,2718- &audio_deliveries,2719- generation,2720- |_| {2721- serde_json::json!({2722- "generation": generation,2723- "available": false,2724- "error": format!("Audio connection ended: {error}")2725- })2726- },2727- );2728- break;2729- }2730- };2731- if !audio_generation_is_current(&connection_generation, generation) {2732- break;2733- }2734- let packet = match AudioPacket::decode_datagram(&datagram) {2735- Ok(packet) => packet,2736- Err(error) => {2737- emit_audio_event_if_current(2738- &app,2739- "audio-state",2740- &connection_generation,2741- &audio_deliveries,2742- generation,2743- |_| {2744- serde_json::json!({2745- "generation": generation,2746- "available": false,2747- "error": format!("Invalid audio packet: {error}")2748- })2749- },2750- );2751- break;2752- }2753- };2754- transport_received_total = transport_received_total.saturating_add(1);2755- let (packets, dropped) = match reorder.insert(packet) {2756- Ok(result) => result,2757- Err(error) => {2758- emit_audio_event_if_current(2759- &app,2760- "audio-state",2761- &connection_generation,2762- &audio_deliveries,2763- generation,2764- |_| {2765- serde_json::json!({2766- "generation": generation,2767- "available": false,2768- "error": error2769- })2770- },2771- );2772- break;2773- }2774- };2775- sequence_dropped_total = sequence_dropped_total.saturating_add(dropped);2776- if dropped > 0 {2777- pending_discontinuity = true;2778- }2779-2780- for ordered in packets {2781- let delivery_id = match lock_audio_deliveries(&audio_deliveries).reserve(generation)2782- {2783- Ok(Some(delivery_id)) => delivery_id,2784- Ok(None) => {2785- frontend_dropped_total = frontend_dropped_total.saturating_add(1);2786- pending_discontinuity = true;2787- continue;2788- }2789- Err(_) => return,2790- };2791- let envelope = match encode_audio_channel_packet(2792- generation,2793- delivery_id,2794- ordered.packet,2795- ordered.discontinuity || pending_discontinuity,2796- ) {2797- Ok(envelope) => envelope,2798- Err(error) => {2799- lock_audio_deliveries(&audio_deliveries)2800- .release_failed_delivery(generation, delivery_id);2801- emit_audio_event_if_current(2802- &app,2803- "audio-state",2804- &connection_generation,2805- &audio_deliveries,2806- generation,2807- |_| {2808- serde_json::json!({2809- "generation": generation,2810- "available": false,2811- "error": error2812- })2813- },2814- );2815- return;2816- }2817- };2818- if !audio_generation_is_current(&connection_generation, generation) {2819- lock_audio_deliveries(&audio_deliveries)2820- .release_failed_delivery(generation, delivery_id);2821- return;2822- }2823- if audio_channel.send(Response::new(envelope)).is_err() {2824- lock_audio_deliveries(&audio_deliveries)2825- .release_failed_delivery(generation, delivery_id);2826- emit_audio_event_if_current(2827- &app,2828- "audio-state",2829- &connection_generation,2830- &audio_deliveries,2831- generation,2832- |_| {2833- serde_json::json!({2834- "generation": generation,2835- "available": false,2836- "error": "Audio webview channel closed"2837- })2838- },2839- );2840- return;2841- }2842- frontend_sent_total = frontend_sent_total.saturating_add(1);2843- pending_discontinuity = false;2844- }2845-2846- if last_stats.elapsed() >= Duration::from_millis(250) {2847- if !emit_audio_event_if_current(2848- &app,2849- "audio-stats",2850- &connection_generation,2851- &audio_deliveries,2852- generation,2853- |deliveries| {2854- serde_json::json!({2855- "generation": generation,2856- "transport_received_total": transport_received_total,2857- "sequence_dropped_total": sequence_dropped_total,2858- "frontend_dropped_total": frontend_dropped_total,2859- "frontend_sent_total": frontend_sent_total,2860- "frontend_queue_depth": deliveries.depth(generation).unwrap_or(0),2861- "frontend_queue_capacity": AUDIO_DELIVERY_CAPACITY,2862- })2863- },2864- ) {2865- return;2866- }2867- last_stats = Instant::now();2868- }2869- }2870- });2871- Ok(audio_connection)2872-}2873-287422 287523 pub async fn iroh_client_connect(287624 app: AppHandle,@@ -2881,2818 +29,16 @@ pub async fn iroh_client_connect(
288129 pointer_channel: Channel<PointerFeedbackPayload>,288230 audio_supported: bool,288331 ) -> Result<ConnectResult, String> {2884- let _connection_serial = state.client_connection_serial.try_lock().map_err(|_| {2885- "Another client connection or disconnection operation is in progress".to_string()2886- })?;2887- let connect_guard = ClientConnectGuard::acquire(Arc::clone(&state.client_connection_active))?;2888-2889- let (client_secret, development_mode) = if let Some(node_id) = state.dev_connect_node_id {2890- if !development_direct_node_available() {2891- return Err(2892- "Development direct-node routing requires a debug build or the explicit demo-direct-node feature"2893- .to_string(),2894- );2895- }2896- let _ = app.emit(2897- "dev-connect-routing",2898- serde_json::json!({2899- "host_node_id": node_id.to_string(),2900- "warning": "Passkey identity lookup skipped; this is not client authorization."2901- }),2902- );2903- (SecretKey::generate(), true)2904- } else {2905- // FIDO2 derivation — 30s timeout so a missing/stuck key surfaces quickly.2906- let client_secret = tokio::time::timeout(2907- Duration::from_secs(30),2908- tokio::task::spawn_blocking(move || derive_iroh_secret_from_key(&pin)),2909- )2910- .await2911- .map_err(|_| "Security key timed out (30s). Make sure your key is connected.".to_string())?2912- .map_err(|e| format!("Task failed: {}", e))?2913- .map_err(|e| format!("FIDO2 error: {:?}", e))?;2914-2915- // Key has been tapped — relay connection is next; update the UI overlay.2916- let _ = app.emit("fido-done", ());2917- (client_secret, false)2918- };2919-2920- let (host_node_id, grants, invitation) = if development_mode {2921- (2922- state2923- .dev_connect_node_id2924- .ok_or_else(|| "Development host routing disappeared".to_string())?,2925- InvitationGrants::ALL,2926- None,2927- )2928- } else {2929- let enrollment = connection_enrollment(&app, client_secret.public())?;2930- (2931- enrollment.host_node_id,2932- enrollment.grants,2933- enrollment.pending_invitation,2934- )2935- };2936- let mut handshake_nonce = [0_u8; 16];2937- getrandom::fill(&mut handshake_nonce)2938- .map_err(|error| format!("Failed to generate handshake nonce: {error}"))?;2939- let endpoint = Endpoint::builder(presets::N0)2940- .secret_key(client_secret)2941- .bind()2942- .await2943- .map_err(|e| format!("Failed to bind endpoint: {}", e))?;2944-2945- let _ = tokio::time::timeout(Duration::from_secs(10), endpoint.online()).await;2946-2947- // Use just the node ID — the presets::N0 relay map handles geographic2948- // routing and fallback across all N0 relays automatically.2949- let addr = iroh::EndpointAddr::new(host_node_id);2950-2951- // Public Sigil authorization exists only on the bounded, negotiated v12952- // protocols. The inherited v0 leg is retained below solely as migration2953- // code and is no longer selected by an ordinary Portal connection.2954- let use_v1 = true;2955- let input_alpn = if use_v1 { INPUT_ALPN_V1 } else { INPUT_ALPN };2956-2957- let (frame_conn, mut frame_recv, media_control_stream, media_negotiation, media_transport) =2958- if use_v1 {2959- let first_attempt = open_negotiated_media_stream(2960- &endpoint,2961- &addr,2962- handshake_nonce,2963- invitation.as_deref(),2964- )2965- .await;2966- let (connection, recv, control, negotiation, transport) = match first_attempt {2967- Ok(result) => result,2968- Err(invitation_error)2969- if invitation.is_some()2970- && invitation_error.contains("Portal peer is not authorized") =>2971- {2972- // Recover only the narrow crash window where Sigil durably2973- // consumed the invitation but Portal did not durably clear it.2974- // The replay itself remains rejected; a second, ticket-free2975- // connection can succeed only as the already-enrolled Iroh2976- // peer authenticated by the exact invited host.2977- open_negotiated_media_stream(&endpoint, &addr, handshake_nonce, None)2978- .await2979- .map_err(|retry_error| {2980- format!(2981- "{invitation_error}; ticket-free enrollment recovery also failed: {retry_error}"2982- )2983- })?2984- }2985- Err(error) => return Err(error),2986- };2987- (connection, recv, control, Some(negotiation), transport)2988- } else {2989- let connection = endpoint@@ -5700,19 +46,7 @@ pub async fn iroh_client_send_input(
@@ -5721,78 +55,7 @@ pub async fn iroh_client_request_keyframe(
@@ -5801,23 +64,7 @@ pub async fn iroh_client_send_media_feedback(
@@ -5825,19 +72,7 @@ pub async fn iroh_client_ack_frame(
@@ -5846,39 +81,7 @@ pub fn iroh_client_ack_audio(
@@ -5886,70 +89,10 @@ pub async fn iroh_client_stop_audio(
diff truncated
@@ -176,10 +176,8 @@ fn reapply_hidden_cursor_rect(
176176 }177177 178178 pub const FRAME_ALPN: &[u8] = sigil_protocol::LEGACY_FRAME_ALPN_V0;179-pub const INPUT_ALPN: &[u8] = sigil_protocol::LEGACY_INPUT_ALPN_V0;180179 pub const RPID: &str = "sigil";181180 pub const SALT_MESSAGE: &str = "sigil-iroh-identity-v1";182-pub const CLIENT_INPUT_QUEUE_CAPACITY: usize = 256;183181 pub const AUDIO_DELIVERY_CAPACITY: usize = 3;184182 185183 @@ -0,0 +1,569 @@
1+use std::sync::atomic::{AtomicU64, Ordering};2+use std::time::Duration;3+4+use iroh::Endpoint;5+use serde::{Deserialize, Serialize};6+use sigil_protocol::{7+ AdaptiveBitrateDecisionV1, AdaptiveBitrateReasonFlagsV1, AdaptiveBitrateStateV1, Capability,8+ MEDIA_FEEDBACK_ALPN_V1, MediaFeedbackFlags, MediaFeedbackReportV1,9+ read_adaptive_bitrate_decision_v1, write_media_feedback_report_v1,10+};11+use tauri::{AppHandle, Emitter, Manager};12+13+use crate::commands::state::{AccumulatedMediaFeedback, AppState};14+use crate::media::frame_channel::take_generation_owned_triple;15+use crate::media::transport::{connect_error_is_unsupported_alpn, negotiate_v1};16+17+pub(crate) const CLIENT_MEDIA_FEEDBACK_CONNECT_TIMEOUT: Duration = Duration::from_secs(3);18+const CLIENT_MEDIA_FEEDBACK_IO_TIMEOUT: Duration = Duration::from_secs(5);19+const CLIENT_ADAPTIVE_DECISION_DELIVERY_INTERVAL: Duration = Duration::from_secs(1);20+21+async fn retire_media_feedback_generation(app: &AppHandle, generation: u64) -> bool {22+ let state = app.state::<AppState>();23+ let connection = {24+ let _connection_serial = state.client_connection_serial.lock().await;25+ let mut slot = state.media_feedback.lock().await;26+ take_generation_owned_triple(&mut slot, generation).map(|(connection, _)| connection)27+ };28+ let Some(connection) = connection else {29+ return false;30+ };31+ connection.close(0_u32.into(), b"adaptive feedback ended");32+ true33+}34+35+36+pub struct ClientMediaFeedbackReport {37+ pub interval_ms: u16,38+ pub last_sequence: Option<u64>,39+ pub transport_dropped_delta: u32,40+ pub frontend_dropped_delta: u32,41+ pub decoder_dropped_delta: u32,42+ pub presenter_dropped_delta: u32,43+ pub frontend_queue_depth: u8,44+ pub frontend_queue_capacity: u8,45+ pub decode_queue_depth: u8,46+ pub decode_queue_capacity: u8,47+ pub presenter_queue_depth: u8,48+ pub presenter_queue_capacity: u8,49+ pub transport_delivery_p95_ms: Option<f64>,50+ pub decode_p95_ms: Option<f64>,51+ pub presentation_p95_ms: Option<f64>,52+ pub resync_active: bool,53+}54+55+56+struct AdaptiveBitrateDecisionPayload {57+ generation: u64,58+ decision: AdaptiveBitrateDecisionDiagnostic,59+}60+61+62+struct AdaptiveBitrateDecisionDiagnostic {63+ decision_id: u64,64+ report_id: u64,65+ target_kbps: u32,66+ floor_kbps: u32,67+ ceiling_kbps: u32,68+ state: &'static str,69+ reasons: Vec<&'static str>,70+ applied: bool,71+}72+73+pub(crate) async fn open_negotiated_feedback_stream(74+ endpoint: &Endpoint,75+ address: &iroh::EndpointAddr,76+ nonce: [u8; 16],77+ media_session_id: u64,78+) -> Result<79+ Option<(80+ iroh::endpoint::Connection,81+ iroh::endpoint::SendStream,82+ iroh::endpoint::RecvStream,83+ )>,84+ String,85+> {86+ let connection = match endpoint87+ .connect(address.clone(), MEDIA_FEEDBACK_ALPN_V1)88+ .await89+ {90+ Ok(connection) => connection,91+ Err(error) if connect_error_is_unsupported_alpn(&error) => return Ok(None),92+ Err(error) => {93+ return Err(format!(94+ "Failed to connect adaptive feedback stream: {error}"95+ ));96+ }97+ };98+ let (mut send, mut recv) = connection99+ .open_bi()100+ .await101+ .map_err(|error| format!("Failed to open adaptive feedback handshake: {error}"))?;102+ let negotiation = negotiate_v1(103+ &mut send,104+ &mut recv,105+ nonce,106+ vec![Capability::VideoH264],107+ Some(Capability::VideoH264),108+ "adaptive feedback",109+ None,110+ )111+ .await?;112+ if negotiation.session_id != media_session_id {113+ return Err("Host returned mismatched media and adaptive feedback sessions".to_string());114+ }115+ Ok(Some((connection, send, recv)))116+}117+118+fn adaptive_bitrate_state_name(state: AdaptiveBitrateStateV1) -> &'static str {119+ match state {120+ AdaptiveBitrateStateV1::Hold => "hold",121+ AdaptiveBitrateStateV1::Decrease => "decrease",122+ AdaptiveBitrateStateV1::Increase => "increase",123+ }124+}125+126+fn adaptive_bitrate_reason_names(reasons: AdaptiveBitrateReasonFlagsV1) -> Vec<&'static str> {127+ [128+ (AdaptiveBitrateReasonFlagsV1::RTT_INFLATION, "rtt-inflation"),129+ (130+ AdaptiveBitrateReasonFlagsV1::LOSS_OR_CANCELLATION,131+ "loss-or-cancellation",132+ ),133+ (134+ AdaptiveBitrateReasonFlagsV1::SENDER_BACKPRESSURE,135+ "sender-backpressure",136+ ),137+ (138+ AdaptiveBitrateReasonFlagsV1::RECEIVER_QUEUE,139+ "receiver-queue",140+ ),141+ (142+ AdaptiveBitrateReasonFlagsV1::DECODE_BACKLOG,143+ "decode-backlog",144+ ),145+ (146+ AdaptiveBitrateReasonFlagsV1::DELIVERY_LATENCY,147+ "delivery-latency",148+ ),149+ (150+ AdaptiveBitrateReasonFlagsV1::CLEAN_RECOVERY,151+ "clean-recovery",152+ ),153+ (154+ AdaptiveBitrateReasonFlagsV1::FEEDBACK_STALE,155+ "feedback-stale",156+ ),157+ ]158+ .into_iter()159+ .filter_map(|(flag, name)| reasons.contains(flag).then_some(name))160+ .collect()161+}162+163+fn adaptive_bitrate_decision_diagnostic(164+ decision: AdaptiveBitrateDecisionV1,165+) -> AdaptiveBitrateDecisionDiagnostic {166+ AdaptiveBitrateDecisionDiagnostic {167+ decision_id: decision.decision_id,168+ report_id: decision.report_id,169+ target_kbps: decision.target_kbps,170+ floor_kbps: decision.floor_kbps,171+ ceiling_kbps: decision.ceiling_kbps,172+ state: adaptive_bitrate_state_name(decision.state),173+ reasons: adaptive_bitrate_reason_names(decision.reasons),174+ applied: decision.applied,175+ }176+}177+178+179+struct AdaptiveDecisionSequence {180+ last_decision_id: Option<u64>,181+ last_report_id: Option<u64>,182+}183+184+impl AdaptiveDecisionSequence {185+ fn accept(&mut self, decision: &AdaptiveBitrateDecisionV1) -> Result<(), String> {186+ if self187+ .last_decision_id188+ .is_some_and(|previous| decision.decision_id <= previous)189+ {190+ return Err(format!(191+ "adaptive decision ID did not increase: previous={:?}, current={}",192+ self.last_decision_id, decision.decision_id193+ ));194+ }195+ if self196+ .last_report_id197+ .is_some_and(|previous| decision.report_id <= previous)198+ {199+ return Err(format!(200+ "adaptive decision report ID did not increase: previous={:?}, current={}",201+ self.last_report_id, decision.report_id202+ ));203+ }204+ self.last_decision_id = Some(decision.decision_id);205+ self.last_report_id = Some(decision.report_id);206+ Ok(())207+ }208+}209+210+async fn emit_paced_adaptive_decisions<F>(211+ generation: u64,212+ mut decisions: tokio::sync::watch::Receiver<Option<AdaptiveBitrateDecisionV1>>,213+ delivery_interval: Duration,214+ mut emit: F,215+) -> Result<(), String>216+where217+ F: FnMut(AdaptiveBitrateDecisionPayload) -> Result<(), String>,218+{219+ let mut delivery = tokio::time::interval(delivery_interval);220+ delivery.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);221+ loop {222+ decisions223+ .changed()224+ .await225+ .map_err(|_| "adaptive decision receiver closed".to_string())?;226+ // The first tick is immediate. Later ticks pace the webview boundary;227+ // any host burst replaces the watch value while this task waits.228+ delivery.tick().await;229+ let Some(decision) = *decisions.borrow_and_update() else {230+ continue;231+ };232+ emit(AdaptiveBitrateDecisionPayload {233+ generation,234+ decision: adaptive_bitrate_decision_diagnostic(decision),235+ })?;236+ }237+}238+239+pub(crate) async fn run_media_feedback_session(240+ app: AppHandle,241+ generation: u64,242+ mut send: iroh::endpoint::SendStream,243+ mut recv: iroh::endpoint::RecvStream,244+ mut reports: tokio::sync::watch::Receiver<Option<AccumulatedMediaFeedback>>,245+) {246+ let (decision_tx, decision_rx) = tokio::sync::watch::channel(None);247+ let writer = async {248+ let mut last_written = None;249+ loop {250+ reports251+ .changed()252+ .await253+ .map_err(|_| "adaptive feedback sender closed".to_string())?;254+ let accumulated = *reports.borrow_and_update();255+ let Some(accumulated) = accumulated else {256+ continue;257+ };258+ let report = accumulated.report_since(last_written);259+ tokio::time::timeout(260+ CLIENT_MEDIA_FEEDBACK_IO_TIMEOUT,261+ write_media_feedback_report_v1(&mut send, &report),262+ )263+ .await264+ .map_err(|_| "adaptive feedback write timed out".to_string())?265+ .map_err(|error| format!("adaptive feedback write failed: {error}"))?;266+ last_written = Some(accumulated);267+ }268+ 269+ Ok::<(), String>(())270+ };271+ let reader = async {272+ let mut sequence = AdaptiveDecisionSequence::default();273+ loop {274+ let decision = read_adaptive_bitrate_decision_v1(&mut recv)275+ .await276+ .map_err(|error| format!("adaptive decision read failed: {error}"))?277+ .ok_or_else(|| "adaptive decision stream closed".to_string())?;278+ sequence.accept(&decision)?;279+ decision_tx.send_replace(Some(decision));280+ }281+ 282+ Ok::<(), String>(())283+ };284+ let decision_app = app.clone();285+ let decision_emitter = emit_paced_adaptive_decisions(286+ generation,287+ decision_rx,288+ CLIENT_ADAPTIVE_DECISION_DELIVERY_INTERVAL,289+ move |payload| {290+ decision_app291+ .emit("adaptive-bitrate-decision", payload)292+ .map_err(|error| format!("adaptive decision event delivery failed: {error}"))293+ },294+ );295+ let terminal = tokio::select! {296+ terminal = writer => terminal,297+ terminal = reader => terminal,298+ terminal = decision_emitter => terminal,299+ };300+ if let Err(error) = terminal {301+ eprintln!("[client] {error}");302+ if let Err(emit_error) = app.emit(303+ "adaptive-feedback-state",304+ serde_json::json!({305+ "generation": generation,306+ "available": false,307+ "error": error,308+ }),309+ ) {310+ eprintln!("[client] adaptive feedback terminal event delivery failed: {emit_error}");311+ }312+ }313+ retire_media_feedback_generation(&app, generation).await;314+}315+316+fn feedback_latency_ms(value: Option<f64>, name: &str) -> Result<Option<u16>, String> {317+ value318+ .map(|value| {319+ if !value.is_finite() || !(0.0..=60_000.0).contains(&value) {320+ return Err(format!("{name} must be finite and between 0 and 60000 ms"));321+ }322+ Ok(value.round() as u16)323+ })324+ .transpose()325+}326+327+fn next_feedback_report_id(counter: &AtomicU64) -> Result<u64, String> {328+ counter329+ .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |current| {330+ current.checked_add(1)331+ })332+ .map(|previous| previous + 1)333+ .map_err(|_| "Adaptive feedback report ID overflowed".to_string())334+}335+336+fn media_feedback_report(337+ report_id: u64,338+ input: ClientMediaFeedbackReport,339+) -> Result<MediaFeedbackReportV1, String> {340+ let flags = if input.resync_active {341+ MediaFeedbackFlags::RESYNC_ACTIVE342+ } else {343+ MediaFeedbackFlags::NONE344+ };345+ let report = MediaFeedbackReportV1 {346+ report_id,347+ interval_ms: input.interval_ms,348+ flags,349+ last_sequence: input.last_sequence,350+ transport_dropped_delta: input.transport_dropped_delta,351+ frontend_dropped_delta: input.frontend_dropped_delta,352+ decoder_dropped_delta: input.decoder_dropped_delta,353+ presenter_dropped_delta: input.presenter_dropped_delta,354+ frontend_queue_depth: input.frontend_queue_depth,355+ frontend_queue_capacity: input.frontend_queue_capacity,356+ decode_queue_depth: input.decode_queue_depth,357+ decode_queue_capacity: input.decode_queue_capacity,358+ presenter_queue_depth: input.presenter_queue_depth,359+ presenter_queue_capacity: input.presenter_queue_capacity,360+ transport_delivery_p95_ms: feedback_latency_ms(361+ input.transport_delivery_p95_ms,362+ "delivery p95",363+ )?,364+ decode_p95_ms: feedback_latency_ms(input.decode_p95_ms, "decode p95")?,365+ presentation_p95_ms: feedback_latency_ms(input.presentation_p95_ms, "presentation p95")?,366+ };367+ report368+ .validate()369+ .map_err(|error| format!("Invalid adaptive feedback report: {error}"))?;370+ Ok(report)371+}372+373+pub(crate) async fn send_media_feedback(374+ state: &AppState,375+ generation: u64,376+ report: ClientMediaFeedbackReport,377+) -> Result<bool, String> {378+ let feedback = state.media_feedback.lock().await;379+ let Some((current_generation, _connection, sender)) = feedback.as_ref() else {380+ return Ok(false);381+ };382+ if *current_generation != generation {383+ return Ok(false);384+ }385+ if sender.receiver_count() == 0 {386+ return Err("Adaptive feedback channel closed".to_string());387+ }388+ let report_id = next_feedback_report_id(&state.media_feedback_report_id)?;389+ let report = media_feedback_report(report_id, report)?;390+ sender.send_modify(|pending| match pending {391+ Some(pending) => pending.merge(report),392+ None => *pending = Some(AccumulatedMediaFeedback::new(report)),393+ });394+ Ok(true)395+}396+397+398+mod tests {399+ use super::*;400+401+ fn client_feedback() -> ClientMediaFeedbackReport {402+ ClientMediaFeedbackReport {403+ interval_ms: 1_000,404+ last_sequence: Some(99),405+ transport_dropped_delta: 1,406+ frontend_dropped_delta: 2,407+ decoder_dropped_delta: 3,408+ presenter_dropped_delta: 4,409+ frontend_queue_depth: 1,410+ frontend_queue_capacity: 4,411+ decode_queue_depth: 1,412+ decode_queue_capacity: 2,413+ presenter_queue_depth: 0,414+ presenter_queue_capacity: 2,415+ transport_delivery_p95_ms: Some(17.4),416+ decode_p95_ms: Some(4.6),417+ presentation_p95_ms: None,418+ resync_active: true,419+ }420+ }421+422+ 423+ fn adaptive_feedback_conversion_is_bounded_and_protocol_valid() {424+ let report = media_feedback_report(7, client_feedback()).unwrap();425+ assert_eq!(report.report_id, 7);426+ assert_eq!(report.interval_ms, 1_000);427+ assert_eq!(report.last_sequence, Some(99));428+ assert_eq!(report.transport_delivery_p95_ms, Some(17));429+ assert_eq!(report.decode_p95_ms, Some(5));430+ assert!(report.flags.contains(MediaFeedbackFlags::RESYNC_ACTIVE));431+ report.validate().unwrap();432+433+ let mut invalid = client_feedback();434+ invalid.decode_queue_depth = 3;435+ assert!(media_feedback_report(8, invalid).is_err());436+ let mut invalid_interval = client_feedback();437+ invalid_interval.interval_ms = 249;438+ assert!(media_feedback_report(9, invalid_interval).is_err());439+ assert!(feedback_latency_ms(Some(f64::INFINITY), "decode").is_err());440+ assert!(feedback_latency_ms(Some(60_001.0), "decode").is_err());441+ }442+443+ 444+ fn feedback_report_ids_are_nonzero_monotonic_and_checked() {445+ let counter = AtomicU64::new(0);446+ assert_eq!(next_feedback_report_id(&counter).unwrap(), 1);447+ assert_eq!(next_feedback_report_id(&counter).unwrap(), 2);448+ assert!(next_feedback_report_id(&AtomicU64::new(u64::MAX)).is_err());449+ }450+451+ 452+ async fn adaptive_feedback_watch_coalesces_latest_state_without_losing_pressure() {453+ let (sender, mut receiver) = tokio::sync::watch::channel(None);454+ let first = media_feedback_report(1, client_feedback()).unwrap();455+ sender.send_replace(Some(AccumulatedMediaFeedback::new(first)));456+ receiver.changed().await.unwrap();457+ let stalled_write = receiver.borrow_and_update().unwrap();458+459+ let mut latest = client_feedback();460+ latest.interval_ms = 1_250;461+ latest.transport_dropped_delta = 9;462+ let second = media_feedback_report(2, latest).unwrap();463+ sender.send_modify(|pending| pending.as_mut().unwrap().merge(second));464+ let mut newest = client_feedback();465+ newest.interval_ms = 1_500;466+ newest.transport_dropped_delta = 11;467+ newest.frontend_queue_depth = 3;468+ let third = media_feedback_report(3, newest).unwrap();469+ sender.send_modify(|pending| pending.as_mut().unwrap().merge(third));470+471+ assert_eq!(stalled_write.report_since(None), first);472+ receiver.changed().await.unwrap();473+ let coalesced = receiver474+ .borrow_and_update()475+ .unwrap()476+ .report_since(Some(stalled_write));477+ assert_eq!(coalesced.report_id, 3);478+ assert_eq!(coalesced.interval_ms, 2_750);479+ assert_eq!(coalesced.transport_dropped_delta, 20);480+ assert_eq!(coalesced.frontend_queue_depth, 3);481+ }482+483+ 484+ fn adaptive_decision_diagnostics_are_explicitly_shadow_state() {485+ let diagnostic = adaptive_bitrate_decision_diagnostic(adaptive_decision(2, 1));486+ assert_eq!(diagnostic.state, "decrease");487+ assert_eq!(diagnostic.reasons, vec!["receiver-queue", "decode-backlog"]);488+ assert!(!diagnostic.applied);489+ }490+491+ fn adaptive_decision(decision_id: u64, report_id: u64) -> AdaptiveBitrateDecisionV1 {492+ AdaptiveBitrateDecisionV1 {493+ decision_id,494+ report_id,495+ target_kbps: 8_000,496+ floor_kbps: 4_000,497+ ceiling_kbps: 20_000,498+ state: AdaptiveBitrateStateV1::Decrease,499+ reasons: AdaptiveBitrateReasonFlagsV1::RECEIVER_QUEUE500+ .union(AdaptiveBitrateReasonFlagsV1::DECODE_BACKLOG),501+ applied: false,502+ }503+ }504+505+ 506+ fn adaptive_decision_sequence_rejects_duplicate_or_regressing_ids() {507+ let mut sequence = AdaptiveDecisionSequence::default();508+ sequence.accept(&adaptive_decision(1, 10)).unwrap();509+ sequence.accept(&adaptive_decision(3, 12)).unwrap();510+ assert!(sequence.accept(&adaptive_decision(3, 13)).is_err());511+512+ let mut report_regression = AdaptiveDecisionSequence::default();513+ report_regression.accept(&adaptive_decision(1, 10)).unwrap();514+ assert!(report_regression.accept(&adaptive_decision(2, 10)).is_err());515+ assert!(report_regression.accept(&adaptive_decision(3, 9)).is_err());516+ }517+518+ 519+ async fn adaptive_decision_delivery_is_paced_latest_value_and_emit_failure_is_terminal() {520+ let (sender, receiver) = tokio::sync::watch::channel(None);521+ let (delivered_tx, mut delivered_rx) = tokio::sync::mpsc::channel(2);522+ let delivery = tokio::spawn(emit_paced_adaptive_decisions(523+ 7,524+ receiver,525+ Duration::from_millis(20),526+ move |payload| {527+ delivered_tx528+ .try_send(payload)529+ .map_err(|error| format!("test delivery failed: {error}"))530+ },531+ ));532+ sender.send_replace(Some(adaptive_decision(1, 1)));533+ sender.send_replace(Some(adaptive_decision(2, 2)));534+ let first = tokio::time::timeout(Duration::from_millis(100), delivered_rx.recv())535+ .await536+ .unwrap()537+ .unwrap();538+ assert_eq!(first.generation, 7);539+ assert_eq!(first.decision.decision_id, 2);540+541+ sender.send_replace(Some(adaptive_decision(3, 3)));542+ assert!(543+ tokio::time::timeout(Duration::from_millis(5), delivered_rx.recv())544+ .await545+ .is_err()546+ );547+ let second = tokio::time::timeout(Duration::from_millis(100), delivered_rx.recv())548+ .await549+ .unwrap()550+ .unwrap();551+ assert_eq!(second.decision.decision_id, 3);552+ delivery.abort();553+554+ let (sender, receiver) = tokio::sync::watch::channel(None);555+ let terminal = tokio::spawn(emit_paced_adaptive_decisions(556+ 7,557+ receiver,558+ Duration::from_millis(1),559+ |_payload| Err("webview channel closed".to_string()),560+ ));561+ sender.send_replace(Some(adaptive_decision(4, 4)));562+ let error = tokio::time::timeout(Duration::from_millis(100), terminal)563+ .await564+ .unwrap()565+ .unwrap()566+ .unwrap_err();567+ assert_eq!(error, "webview channel closed");568+ }569+}@@ -0,0 +1,551 @@
1+use std::collections::BTreeMap;2+use std::sync::atomic::{AtomicU64, Ordering};3+use std::sync::{Arc, Mutex as StdMutex, MutexGuard as StdMutexGuard};4+use std::time::{Duration, Instant};5+6+use iroh::Endpoint;7+use sigil_protocol::{AUDIO_ALPN_V1, AudioFlags, AudioPacket, AudioPacketHeader, Capability};8+use tauri::{9+ AppHandle, Emitter, State,10+ ipc::{Channel, Response},11+};12+13+use super::transport::negotiate_v1;14+use crate::commands::state::{AUDIO_DELIVERY_CAPACITY, AppState, AudioDeliveryState};15+16+// Three 20 ms Opus packets cap the Rust→webview handoff at 60 ms. The17+// AudioWorklet owns a separate fixed ring and never feeds back into transport.18+const AUDIO_CHANNEL_MAGIC: [u8; 4] = *b"SGAC";19+const AUDIO_CHANNEL_VERSION: u16 = 1;20+const AUDIO_CHANNEL_HEADER_LEN: usize = 24;21+22+pub(crate) fn lock_audio_deliveries(23+ deliveries: &StdMutex<AudioDeliveryState>,24+) -> StdMutexGuard<'_, AudioDeliveryState> {25+ deliveries26+ .lock()27+ .unwrap_or_else(std::sync::PoisonError::into_inner)28+}29+30+pub(crate) fn next_audio_generation(counter: &AtomicU64) -> Result<u64, String> {31+ counter32+ .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |current| {33+ current.checked_add(1)34+ })35+ .map(|previous| previous + 1)36+ .map_err(|_| "Audio connection generation overflowed".to_string())37+}38+39+fn audio_generation_is_current(counter: &AtomicU64, generation: u64) -> bool {40+ counter.load(Ordering::SeqCst) == generation41+}42+43+fn emit_audio_event_if_current(44+ app: &AppHandle,45+ event: &str,46+ generation_counter: &AtomicU64,47+ deliveries: &StdMutex<AudioDeliveryState>,48+ generation: u64,49+ payload: impl FnOnce(&AudioDeliveryState) -> serde_json::Value,50+) -> bool {51+ let deliveries = lock_audio_deliveries(deliveries);52+ if generation_counter.load(Ordering::SeqCst) != generation53+ || deliveries.generation() != Some(generation)54+ {55+ return false;56+ }57+ let _ = app.emit(event, payload(&deliveries));58+ true59+}60+61+62+struct AudioReorderBuffer {63+ expected_sequence: Option<u64>,64+ packets: BTreeMap<u64, AudioPacket>,65+}66+67+68+struct OrderedAudioPacket {69+ packet: AudioPacket,70+ discontinuity: bool,71+}72+73+impl AudioReorderBuffer {74+ const CAPACITY: usize = 3;75+76+ fn insert(&mut self, packet: AudioPacket) -> Result<(Vec<OrderedAudioPacket>, u64), String> {77+ let sequence = packet.header.sequence;78+ let expected = self.expected_sequence.get_or_insert(sequence);79+ if sequence < *expected || self.packets.contains_key(&sequence) {80+ return Ok((Vec::new(), 0));81+ }82+ self.packets.insert(sequence, packet);83+84+ let mut dropped = 0_u64;85+ let mut discontinuity = false;86+ if !self.packets.contains_key(expected) && self.packets.len() >= Self::CAPACITY {87+ let next = *self88+ .packets89+ .first_key_value()90+ .expect("capacity check guarantees one packet")91+ .0;92+ dropped = next.saturating_sub(*expected);93+ *expected = next;94+ discontinuity = true;95+ }96+97+ let mut ordered = Vec::with_capacity(Self::CAPACITY);98+ while let Some(packet) = self.packets.remove(expected) {99+ ordered.push(OrderedAudioPacket {100+ packet,101+ discontinuity,102+ });103+ discontinuity = false;104+ *expected = expected105+ .checked_add(1)106+ .ok_or_else(|| "Audio sequence overflowed".to_string())?;107+ }108+ Ok((ordered, dropped))109+ }110+}111+112+fn encode_audio_channel_packet(113+ generation: u64,114+ delivery_id: u64,115+ packet: AudioPacket,116+ force_discontinuity: bool,117+) -> Result<Vec<u8>, String> {118+ let protocol_packet = if force_discontinuity {119+ let header = AudioPacketHeader::opus(120+ packet.payload.len(),121+ packet.header.sequence,122+ packet.header.capture_timestamp_us,123+ packet.header.pts_us,124+ AudioFlags::DISCONTINUITY,125+ )126+ .map_err(|error| error.to_string())?;127+ AudioPacket::new(header, packet.payload).map_err(|error| error.to_string())?128+ } else {129+ packet130+ };131+ let datagram = protocol_packet132+ .encode_datagram()133+ .map_err(|error| error.to_string())?;134+ let mut envelope = Vec::with_capacity(AUDIO_CHANNEL_HEADER_LEN + datagram.len());135+ envelope.extend_from_slice(&AUDIO_CHANNEL_MAGIC);136+ envelope.extend_from_slice(&AUDIO_CHANNEL_VERSION.to_be_bytes());137+ envelope.extend_from_slice(&(AUDIO_CHANNEL_HEADER_LEN as u16).to_be_bytes());138+ envelope.extend_from_slice(&generation.to_be_bytes());139+ envelope.extend_from_slice(&delivery_id.to_be_bytes());140+ envelope.extend_from_slice(&datagram);141+ debug_assert_eq!(envelope.len(), AUDIO_CHANNEL_HEADER_LEN + datagram.len());142+ Ok(envelope)143+}144+145+pub(crate) struct AudioStartRequest {146+ pub(crate) address: iroh::EndpointAddr,147+ pub(crate) handshake_nonce: [u8; 16],148+ pub(crate) media_session_id: Option<u64>,149+ pub(crate) audio_supported: bool,150+ pub(crate) audio_channel: Channel<Response>,151+ pub(crate) audio_deliveries: Arc<StdMutex<AudioDeliveryState>>,152+ pub(crate) connection_generation: Arc<AtomicU64>,153+ pub(crate) generation: u64,154+}155+156+pub(crate) async fn try_start_audio(157+ app: AppHandle,158+ endpoint: &Endpoint,159+ request: AudioStartRequest,160+) -> Result<iroh::endpoint::Connection, String> {161+ if !request.audio_supported {162+ return Err("WebCodecs Opus AudioDecoder is unavailable".to_string());163+ }164+ let media_session_id = request165+ .media_session_id166+ .ok_or_else(|| "The connected host protocol does not negotiate audio".to_string())?;167+ let audio_connection = tokio::time::timeout(168+ Duration::from_secs(3),169+ endpoint.connect(request.address, AUDIO_ALPN_V1),170+ )171+ .await172+ .map_err(|_| "Timed out connecting optional audio".to_string())?173+ .map_err(|error| format!("Audio connection unavailable: {error}"))?;174+ let (mut send, mut recv) = audio_connection175+ .open_bi()176+ .await177+ .map_err(|error| format!("Failed to open audio handshake: {error}"))?;178+ let negotiation = negotiate_v1(179+ &mut send,180+ &mut recv,181+ request.handshake_nonce,182+ vec![Capability::AudioOpus],183+ Some(Capability::AudioOpus),184+ "audio",185+ None,186+ )187+ .await?;188+ if negotiation.session_id != media_session_id {189+ audio_connection.close(1_u32.into(), b"audio session mismatch");190+ return Err("Host returned mismatched media and audio sessions".to_string());191+ }192+ send.finish()193+ .map_err(|error| format!("Failed to finish audio handshake: {error}"))?;194+195+ let task_audio_connection = audio_connection.clone();196+ tokio::spawn(async move {197+ let audio_channel = request.audio_channel;198+ let audio_deliveries = request.audio_deliveries;199+ let connection_generation = request.connection_generation;200+ let generation = request.generation;201+ let mut reorder = AudioReorderBuffer::default();202+ let mut transport_received_total = 0_u64;203+ let mut sequence_dropped_total = 0_u64;204+ let mut frontend_dropped_total = 0_u64;205+ let mut frontend_sent_total = 0_u64;206+ let mut pending_discontinuity = false;207+ let mut last_stats = Instant::now();208+209+ loop {210+ let datagram = match task_audio_connection.read_datagram().await {211+ Ok(datagram) => datagram,212+ Err(error) => {213+ emit_audio_event_if_current(214+ &app,215+ "audio-state",216+ &connection_generation,217+ &audio_deliveries,218+ generation,219+ |_| {220+ serde_json::json!({221+ "generation": generation,222+ "available": false,223+ "error": format!("Audio connection ended: {error}")224+ })225+ },226+ );227+ break;228+ }229+ };230+ if !audio_generation_is_current(&connection_generation, generation) {231+ break;232+ }233+ let packet = match AudioPacket::decode_datagram(&datagram) {234+ Ok(packet) => packet,235+ Err(error) => {236+ emit_audio_event_if_current(237+ &app,238+ "audio-state",239+ &connection_generation,240+ &audio_deliveries,241+ generation,242+ |_| {243+ serde_json::json!({244+ "generation": generation,245+ "available": false,246+ "error": format!("Invalid audio packet: {error}")247+ })248+ },249+ );250+ break;251+ }252+ };253+ transport_received_total = transport_received_total.saturating_add(1);254+ let (packets, dropped) = match reorder.insert(packet) {255+ Ok(result) => result,256+ Err(error) => {257+ emit_audio_event_if_current(258+ &app,259+ "audio-state",260+ &connection_generation,261+ &audio_deliveries,262+ generation,263+ |_| {264+ serde_json::json!({265+ "generation": generation,266+ "available": false,267+ "error": error268+ })269+ },270+ );271+ break;272+ }273+ };274+ sequence_dropped_total = sequence_dropped_total.saturating_add(dropped);275+ if dropped > 0 {276+ pending_discontinuity = true;277+ }278+279+ for ordered in packets {280+ let delivery_id = match lock_audio_deliveries(&audio_deliveries).reserve(generation)281+ {282+ Ok(Some(delivery_id)) => delivery_id,283+ Ok(None) => {284+ frontend_dropped_total = frontend_dropped_total.saturating_add(1);285+ pending_discontinuity = true;286+ continue;287+ }288+ Err(_) => return,289+ };290+ let envelope = match encode_audio_channel_packet(291+ generation,292+ delivery_id,293+ ordered.packet,294+ ordered.discontinuity || pending_discontinuity,295+ ) {296+ Ok(envelope) => envelope,297+ Err(error) => {298+ lock_audio_deliveries(&audio_deliveries)299+ .release_failed_delivery(generation, delivery_id);300+ emit_audio_event_if_current(301+ &app,302+ "audio-state",303+ &connection_generation,304+ &audio_deliveries,305+ generation,306+ |_| {307+ serde_json::json!({308+ "generation": generation,309+ "available": false,310+ "error": error311+ })312+ },313+ );314+ return;315+ }316+ };317+ if !audio_generation_is_current(&connection_generation, generation) {318+ lock_audio_deliveries(&audio_deliveries)319+ .release_failed_delivery(generation, delivery_id);320+ return;321+ }322+ if audio_channel.send(Response::new(envelope)).is_err() {323+ lock_audio_deliveries(&audio_deliveries)324+ .release_failed_delivery(generation, delivery_id);325+ emit_audio_event_if_current(326+ &app,327+ "audio-state",328+ &connection_generation,329+ &audio_deliveries,330+ generation,331+ |_| {332+ serde_json::json!({333+ "generation": generation,334+ "available": false,335+ "error": "Audio webview channel closed"336+ })337+ },338+ );339+ return;340+ }341+ frontend_sent_total = frontend_sent_total.saturating_add(1);342+ pending_discontinuity = false;343+ }344+345+ if last_stats.elapsed() >= Duration::from_millis(250) {346+ if !emit_audio_event_if_current(347+ &app,348+ "audio-stats",349+ &connection_generation,350+ &audio_deliveries,351+ generation,352+ |deliveries| {353+ serde_json::json!({354+ "generation": generation,355+ "transport_received_total": transport_received_total,356+ "sequence_dropped_total": sequence_dropped_total,357+ "frontend_dropped_total": frontend_dropped_total,358+ "frontend_sent_total": frontend_sent_total,359+ "frontend_queue_depth": deliveries.depth(generation).unwrap_or(0),360+ "frontend_queue_capacity": AUDIO_DELIVERY_CAPACITY,361+ })362+ },363+ ) {364+ return;365+ }366+ last_stats = Instant::now();367+ }368+ }369+ });370+ Ok(audio_connection)371+}372+373+pub(crate) fn iroh_client_ack_audio(374+ state: State<'_, AppState>,375+ generation: u64,376+ delivery_id: u64,377+) -> Result<bool, String> {378+ lock_audio_deliveries(&state.audio_deliveries).acknowledge(generation, delivery_id)?;379+ Ok(true)380+}381+382+pub(crate) fn cancel_audio_generation(383+ generation_counter: &AtomicU64,384+ deliveries: &StdMutex<AudioDeliveryState>,385+ expected_generation: u64,386+) -> Result<bool, String> {387+ let replacement_generation = expected_generation388+ .checked_add(1)389+ .ok_or_else(|| "Audio connection generation overflowed".to_string())?;390+ let mut deliveries = lock_audio_deliveries(deliveries);391+ if deliveries.generation() != Some(expected_generation)392+ || generation_counter.load(Ordering::SeqCst) != expected_generation393+ {394+ return Ok(false);395+ }396+ if generation_counter397+ .compare_exchange(398+ expected_generation,399+ replacement_generation,400+ Ordering::SeqCst,401+ Ordering::SeqCst,402+ )403+ .is_err()404+ {405+ return Ok(false);406+ }407+ if !deliveries.cancel_generation(expected_generation) {408+ return Err("Audio delivery generation changed during cancellation".to_string());409+ }410+ Ok(true)411+}412+413+pub(crate) async fn iroh_client_stop_audio(414+ state: State<'_, AppState>,415+ expected_generation: u64,416+) -> Result<bool, String> {417+ if !cancel_audio_generation(418+ &state.audio_connection_generation,419+ &state.audio_deliveries,420+ expected_generation,421+ )? {422+ return Ok(false);423+ }424+425+ let connection = {426+ let mut audio_connection = state.audio_connection.lock().await;427+ if audio_connection428+ .as_ref()429+ .is_some_and(|(generation, _)| *generation == expected_generation)430+ {431+ audio_connection.take().map(|(_, connection)| connection)432+ } else {433+ None434+ }435+ };436+ if let Some(connection) = connection {437+ connection.close(0_u32.into(), b"audio stopped by client");438+ }439+ Ok(true)440+}441+442+443+mod tests {444+ use super::*;445+446+ fn audio_packet(sequence: u64) -> AudioPacket {447+ let payload = vec![0xf8, 0xff, 0xfe];448+ AudioPacket::new(449+ AudioPacketHeader::opus(450+ payload.len(),451+ sequence,452+ sequence * 20_000,453+ sequence as i64 * 20_000,454+ AudioFlags::NONE,455+ )456+ .unwrap(),457+ payload,458+ )459+ .unwrap()460+ }461+462+ 463+ fn audio_delivery_tokens_are_bounded_unique_and_acknowledged_exactly_once() {464+ let mut deliveries = AudioDeliveryState::default();465+ deliveries.begin_generation(7).unwrap();466+467+ assert_eq!(deliveries.reserve(7).unwrap(), Some(1));468+ assert_eq!(deliveries.reserve(7).unwrap(), Some(2));469+ assert_eq!(deliveries.reserve(7).unwrap(), Some(3));470+ assert_eq!(deliveries.depth(7), Some(AUDIO_DELIVERY_CAPACITY));471+ assert_eq!(deliveries.reserve(7).unwrap(), None);472+473+ deliveries.acknowledge(7, 2).unwrap();474+ assert!(deliveries.acknowledge(7, 2).is_err());475+ assert_eq!(deliveries.reserve(7).unwrap(), Some(4));476+ assert!(deliveries.acknowledge(6, 1).is_err());477+ assert!(deliveries.acknowledge(7, 99).is_err());478+ assert_eq!(deliveries.depth(7), Some(AUDIO_DELIVERY_CAPACITY));479+ }480+481+ 482+ fn audio_generation_cancellation_rejects_stale_and_clears_current_tokens() {483+ let counter = AtomicU64::new(11);484+ let deliveries = StdMutex::new(AudioDeliveryState::default());485+ lock_audio_deliveries(&deliveries)486+ .begin_generation(11)487+ .unwrap();488+ assert_eq!(489+ lock_audio_deliveries(&deliveries).reserve(11).unwrap(),490+ Some(1)491+ );492+493+ assert!(!cancel_audio_generation(&counter, &deliveries, 10).unwrap());494+ assert_eq!(counter.load(Ordering::SeqCst), 11);495+ assert_eq!(lock_audio_deliveries(&deliveries).depth(11), Some(1));496+497+ assert!(cancel_audio_generation(&counter, &deliveries, 11).unwrap());498+ assert_eq!(counter.load(Ordering::SeqCst), 12);499+ assert_eq!(lock_audio_deliveries(&deliveries).generation(), None);500+ assert!(!cancel_audio_generation(&counter, &deliveries, 11).unwrap());501+ }502+503+ 504+ fn audio_reorder_window_is_bounded_and_marks_skipped_packets() {505+ let mut reorder = AudioReorderBuffer::default();506+ let (first, dropped) = reorder.insert(audio_packet(10)).unwrap();507+ assert_eq!(first.len(), 1);508+ assert_eq!(dropped, 0);509+510+ assert!(reorder.insert(audio_packet(12)).unwrap().0.is_empty());511+ assert!(reorder.insert(audio_packet(13)).unwrap().0.is_empty());512+ let (ordered, dropped) = reorder.insert(audio_packet(14)).unwrap();513+ assert_eq!(dropped, 1);514+ assert_eq!(515+ ordered516+ .iter()517+ .map(|packet| packet.packet.header.sequence)518+ .collect::<Vec<_>>(),519+ vec![12, 13, 14]520+ );521+ assert!(ordered[0].discontinuity);522+ assert!(!ordered[1].discontinuity);523+ assert!(reorder.packets.len() <= AudioReorderBuffer::CAPACITY);524+525+ assert!(reorder.insert(audio_packet(16)).unwrap().0.is_empty());526+ let (ordered, dropped) = reorder.insert(audio_packet(15)).unwrap();527+ assert_eq!(dropped, 0);528+ assert_eq!(529+ ordered530+ .iter()531+ .map(|packet| packet.packet.header.sequence)532+ .collect::<Vec<_>>(),533+ vec![15, 16]534+ );535+ assert!(reorder.insert(audio_packet(16)).unwrap().0.is_empty());536+ }537+538+ 539+ fn audio_channel_envelope_is_protocol_strict_and_can_force_discontinuity() {540+ let encoded = encode_audio_channel_packet(9, 42, audio_packet(7), true).unwrap();541+ assert_eq!(&encoded[0..4], b"SGAC");542+ assert_eq!(&encoded[4..6], &1_u16.to_be_bytes());543+ assert_eq!(&encoded[6..8], &24_u16.to_be_bytes());544+ assert_eq!(&encoded[8..16], &9_u64.to_be_bytes());545+ assert_eq!(&encoded[16..24], &42_u64.to_be_bytes());546+ let decoded = AudioPacket::decode_datagram(&encoded[AUDIO_CHANNEL_HEADER_LEN..]).unwrap();547+ assert_eq!(decoded.header.sequence, 7);548+ assert!(decoded.header.flags.contains(AudioFlags::DISCONTINUITY));549+ assert_eq!(decoded.payload, vec![0xf8, 0xff, 0xfe]);550+ }551+}@@ -0,0 +1,516 @@
1+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};2+use std::sync::{Arc, Mutex as StdMutex};3+use std::time::{Duration, Instant};4+5+use iroh::{Endpoint, SecretKey, endpoint::presets};6+use serde::Serialize;7+use sigil_protocol::{8+ InputEvent, InvitationGrants, KeyframeRequestReasonV3, PointerSurfaceDimensions,9+};10+use tauri::{11+ AppHandle, Emitter,12+ ipc::{Channel, Response},13+};14+15+use crate::commands::auth::derive_iroh_secret_from_key;16+use crate::commands::enrollment::{connection_enrollment, mark_invitation_redeemed};17+use crate::commands::state::{18+ AppState, FRAME_ALPN, MediaFeedbackSender, development_direct_node_available,19+};20+use crate::media::adaptive_feedback::{21+ CLIENT_MEDIA_FEEDBACK_CONNECT_TIMEOUT, open_negotiated_feedback_stream,22+ run_media_feedback_session,23+};24+use crate::media::audio_delivery::{25+ AudioStartRequest, lock_audio_deliveries, next_audio_generation, try_start_audio,26+};27+use crate::media::frame_channel::{close_generation_connection, next_media_generation};28+use crate::media::input_delivery::{29+ CLIENT_INPUT_QUEUE_CAPACITY, InputSession, PointerFeedbackPayload, open_input_session,30+ run_input_feedback, run_input_forwarder,31+};32+use crate::media::media_control::run_media_control_writer_v3;33+use crate::media::moq_receiver::open_upstream_moq_media;34+use crate::media::network_diagnostics::NetworkSessionDiagnostics;35+use crate::media::transport::{36+ CLIENT_ENDPOINT_CLOSE_TIMEOUT, MediaTransport, open_negotiated_media_stream,37+};38+use crate::media::video_delivery::{VideoDeliveryRequest, run_video_delivery};39+40+struct ClientConnectGuard {41+ active: Arc<AtomicBool>,42+ committed: bool,43+}44+45+impl ClientConnectGuard {46+ fn acquire(active: Arc<AtomicBool>) -> Result<Self, String> {47+ active48+ .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)49+ .map_err(|_| "A client connection is already active or in progress".to_string())?;50+ Ok(Self {51+ active,52+ committed: false,53+ })54+ }55+56+ fn commit(mut self) {57+ self.committed = true;58+ }59+}60+61+impl Drop for ClientConnectGuard {62+ fn drop(&mut self) {63+ if !self.committed {64+ self.active.store(false, Ordering::SeqCst);65+ }66+ }67+}68+69+70+pub struct ConnectResult {71+ pub connected: bool,72+ pub host_node_id: Option<String>,73+ pub development_mode: bool,74+ pub media_transport: &'static str,75+ pub pointer_surface_dimensions: Option<PointerSurfaceDimensions>,76+ pub relative_pointer_available: bool,77+ pub pointer_position_feedback_available: bool,78+ pub absolute_pointer_available: bool,79+ pub keyboard_available: bool,80+ pub text_available: bool,81+ pub gamepad_available: bool,82+ pub control_available: bool,83+ pub audio_available: bool,84+ pub audio_generation: Option<u64>,85+ pub audio_error: Option<String>,86+ pub media_generation: u64,87+ pub adaptive_feedback_available: bool,88+ pub adaptive_feedback_error: Option<String>,89+ pub error: Option<String>,90+}91+92+pub(crate) async fn connect_client(93+ app: AppHandle,94+ state: &AppState,95+ pin: String,96+ frame_channel: Channel<Response>,97+ audio_channel: Channel<Response>,98+ pointer_channel: Channel<PointerFeedbackPayload>,99+ audio_supported: bool,100+) -> Result<ConnectResult, String> {101+ let _connection_serial = state.client_connection_serial.try_lock().map_err(|_| {102+ "Another client connection or disconnection operation is in progress".to_string()103+ })?;104+ let connect_guard = ClientConnectGuard::acquire(Arc::clone(&state.client_connection_active))?;105+106+ let (client_secret, development_mode) = if let Some(node_id) = state.dev_connect_node_id {107+ if !development_direct_node_available() {108+ return Err(109+ "Development direct-node routing requires a debug build or the explicit demo-direct-node feature"110+ .to_string(),111+ );112+ }113+ let _ = app.emit(114+ "dev-connect-routing",115+ serde_json::json!({116+ "host_node_id": node_id.to_string(),117+ "warning": "Passkey identity lookup skipped; this is not client authorization."118+ }),119+ );120+ (SecretKey::generate(), true)121+ } else {122+ // FIDO2 derivation — 30s timeout so a missing/stuck key surfaces quickly.123+ let client_secret = tokio::time::timeout(124+ Duration::from_secs(30),125+ tokio::task::spawn_blocking(move || derive_iroh_secret_from_key(&pin)),126+ )127+ .await128+ .map_err(|_| "Security key timed out (30s). Make sure your key is connected.".to_string())?129+ .map_err(|e| format!("Task failed: {}", e))?130+ .map_err(|e| format!("FIDO2 error: {:?}", e))?;131+132+ // Key has been tapped — relay connection is next; update the UI overlay.133+ let _ = app.emit("fido-done", ());134+ (client_secret, false)135+ };136+137+ let (host_node_id, grants, invitation) = if development_mode {138+ (139+ state140+ .dev_connect_node_id141+ .ok_or_else(|| "Development host routing disappeared".to_string())?,142+ InvitationGrants::ALL,143+ None,144+ )145+ } else {146+ let enrollment = connection_enrollment(&app, client_secret.public())?;147+ (148+ enrollment.host_node_id,149+ enrollment.grants,150+ enrollment.pending_invitation,151+ )152+ };153+ let mut handshake_nonce = [0_u8; 16];154+ getrandom::fill(&mut handshake_nonce)155+ .map_err(|error| format!("Failed to generate handshake nonce: {error}"))?;156+ let endpoint = Endpoint::builder(presets::N0)157+ .secret_key(client_secret)158+ .bind()159+ .await160+ .map_err(|e| format!("Failed to bind endpoint: {}", e))?;161+162+ let _ = tokio::time::timeout(Duration::from_secs(10), endpoint.online()).await;163+164+ // Use just the node ID — the presets::N0 relay map handles geographic165+ // routing and fallback across all N0 relays automatically.166+ let addr = iroh::EndpointAddr::new(host_node_id);167+168+ // Public Sigil authorization exists only on the bounded, negotiated v1169+ // protocols. The inherited v0 leg is retained below solely as migration170+ // code and is no longer selected by an ordinary Portal connection.171+ let use_v1 = true;172+ let (frame_conn, frame_recv, media_control_stream, media_negotiation, media_transport) =173+ if use_v1 {174+ let first_attempt = open_negotiated_media_stream(175+ &endpoint,176+ &addr,177+ handshake_nonce,178+ invitation.as_deref(),179+ )180+ .await;181+ let (connection, recv, control, negotiation, transport) = match first_attempt {182+ Ok(result) => result,183+ Err(invitation_error)184+ if invitation.is_some()185+ && invitation_error.contains("Portal peer is not authorized") =>186+ {187+ // Recover only the narrow crash window where Sigil durably188+ // consumed the invitation but Portal did not durably clear it.189+ // The replay itself remains rejected; a second, ticket-free190+ // connection can succeed only as the already-enrolled Iroh191+ // peer authenticated by the exact invited host.192+ open_negotiated_media_stream(&endpoint, &addr, handshake_nonce, None)193+ .await194+ .map_err(|retry_error| {195+ format!(196+ "{invitation_error}; ticket-free enrollment recovery also failed: {retry_error}"197+ )198+ })?199+ }200+ Err(error) => return Err(error),201+ };202+ (connection, recv, control, Some(negotiation), transport)203+ } else {204+ let connection = endpoint205+ .connect(addr.clone(), FRAME_ALPN)206+ .await207+ .map_err(|e| format!("Failed to connect frame stream: {e}"))?;208+ let (mut send, recv) = connection209+ .open_bi()210+ .await211+ .map_err(|e| format!("Failed to open frame stream: {e}"))?;212+ send.write_all(&[1u8])213+ .await214+ .map_err(|e| format!("Failed to send start: {e}"))?;215+ send.finish()216+ .map_err(|e| format!("Failed to finish frame start stream: {e}"))?;217+ (connection, recv, None, None, MediaTransport::LegacyV0)218+ };219+ let media_session_id = media_negotiation220+ .as_ref()221+ .map(|negotiation| negotiation.session_id);222+ let (upstream_moq_media, frame_connection_for_stats) = if media_transport223+ == MediaTransport::UpstreamMoq224+ {225+ let session_id = media_session_id226+ .ok_or_else(|| "Host omitted the control session ID required by MoQ".to_string())?;227+ let (receiver, diagnostics_connection) =228+ match open_upstream_moq_media(&endpoint, &addr, session_id).await {229+ Ok(media) => media,230+ Err(error) => {231+ // CONTROL already authenticated and owns the host's232+ // one-client lease. A post-auth MoQ failure is233+ // terminal, and must explicitly release that lease;234+ // it must never fall through to a legacy media ALPN.235+ frame_conn.close(1_u32.into(), b"upstream MoQ setup failed");236+ let _ =237+ tokio::time::timeout(CLIENT_ENDPOINT_CLOSE_TIMEOUT, endpoint.close()).await;238+ return Err(error);239+ }240+ };241+ (Some(receiver), diagnostics_connection)242+ } else {243+ (None, frame_conn.clone())244+ };245+ let pointer_surface_dimensions = media_negotiation246+ .as_ref()247+ .and_then(|negotiation| negotiation.pointer_surface_dimensions);248+ if !development_mode && let Some(expected_invitation) = invitation.as_deref() {249+ // An accepted media hello means Sigil durably committed the one-time250+ // enrollment before returning. Future PIN/tap/play sessions send no251+ // bearer credential and authenticate by the stable Iroh peer instead.252+ mark_invitation_redeemed(&app, expected_invitation)?;253+ }254+255+ // Feedback is a v3 sidecar for both the preferred upstream-MoQ transport256+ // and the grouped-v3 compatibility path. Unsupported ALPN is normal257+ // compatibility with older Sigil hosts; all other failures remain visible258+ // diagnostics but never downgrade the authenticated media session.259+ let (adaptive_feedback_stream, adaptive_feedback_error) = if media_transport260+ .supports_adaptive_feedback()261+ {262+ match tokio::time::timeout(263+ CLIENT_MEDIA_FEEDBACK_CONNECT_TIMEOUT,264+ open_negotiated_feedback_stream(265+ &endpoint,266+ &addr,267+ handshake_nonce,268+ media_session_id.ok_or_else(|| "Media v3 omitted its session ID".to_string())?,269+ ),270+ )271+ .await272+ {273+ Ok(Ok(stream)) => (stream, None),274+ Ok(Err(error)) => {275+ eprintln!("[client] adaptive feedback unavailable: {error}");276+ (None, Some(error))277+ }278+ Err(_) => {279+ let error = "Adaptive feedback negotiation timed out".to_string();280+ eprintln!("[client] adaptive feedback unavailable: {error}");281+ (None, Some(error))282+ }283+ }284+ } else {285+ (None, None)286+ };287+288+ let InputSession {289+ connection: input_connection,290+ send: input_send,291+ recv: input_recv,292+ capabilities: input_capabilities,293+ availability: input_availability,294+ } = open_input_session(295+ &endpoint,296+ &addr,297+ handshake_nonce,298+ media_session_id,299+ grants,300+ use_v1,301+ )302+ .await?;303+304+ let network_diagnostics = Arc::new(StdMutex::new(NetworkSessionDiagnostics::new(305+ Instant::now(),306+ input_availability.input_ack,307+ )));308+309+ if input_availability.pointer_position_feedback || input_availability.input_ack {310+ let feedback_diagnostics = Arc::clone(&network_diagnostics);311+ let pointer_feedback_enabled = input_availability.pointer_position_feedback;312+ let input_ack_enabled = input_availability.input_ack;313+ tokio::spawn(run_input_feedback(314+ input_recv,315+ feedback_diagnostics,316+ pointer_feedback_enabled,317+ input_ack_enabled,318+ pointer_channel,319+ ));320+ } else {321+ drop(input_recv);322+ }323+324+ let audio_generation = next_audio_generation(&state.audio_connection_generation)?;325+ lock_audio_deliveries(&state.audio_deliveries).begin_generation(audio_generation)?;326+ let audio_result = try_start_audio(327+ app.clone(),328+ &endpoint,329+ AudioStartRequest {330+ address: addr.clone(),331+ handshake_nonce,332+ media_session_id,333+ audio_supported,334+ audio_channel,335+ audio_deliveries: Arc::clone(&state.audio_deliveries),336+ connection_generation: Arc::clone(&state.audio_connection_generation),337+ generation: audio_generation,338+ },339+ )340+ .await;341+ let mut audio_connection_for_stats = None;342+ let (audio_available, connected_audio_generation, audio_error) = match audio_result {343+ Ok(connection) => {344+ audio_connection_for_stats = Some(connection.clone());345+ *state.audio_connection.lock().await = Some((audio_generation, connection));346+ (true, Some(audio_generation), None)347+ }348+ Err(error) => {349+ lock_audio_deliveries(&state.audio_deliveries).cancel_generation(audio_generation);350+ (false, None, Some(error))351+ }352+ };353+ let media_generation = next_media_generation(&state.client_media_generation)?;354+355+ let (tx, rx) = tokio::sync::mpsc::channel::<InputEvent>(CLIENT_INPUT_QUEUE_CAPACITY);356+ {357+ let mut input_send_guard = state.input_send.lock().await;358+ *input_send_guard = Some(tx);359+ }360+361+ {362+ let mut ce = state.client_endpoint.lock().await;363+ *ce = Some(endpoint.clone());364+ }365+ *state.media_connection.lock().await = Some((media_generation, frame_conn.clone()));366+ let adaptive_feedback_available = adaptive_feedback_stream.is_some();367+ if let Some((connection, send, recv)) = adaptive_feedback_stream {368+ let (feedback_tx, feedback_rx) = tokio::sync::watch::channel(None);369+ let feedback_sender: MediaFeedbackSender = feedback_tx;370+ *state.media_feedback.lock().await =371+ Some((media_generation, connection, feedback_sender.clone()));372+ tokio::spawn(run_media_feedback_session(373+ app.clone(),374+ media_generation,375+ send,376+ recv,377+ feedback_rx,378+ ));379+ } else {380+ *state.media_feedback.lock().await = None;381+ }382+ let media_control_requests = if let Some(control_stream) = media_control_stream {383+ let (control_tx, control_rx) = tokio::sync::mpsc::channel(1);384+ *state.media_control.lock().await = Some((media_generation, control_tx.clone()));385+ tokio::spawn(run_media_control_writer_v3(control_stream, control_rx));386+ if media_transport == MediaTransport::GroupedObjectsV3 {387+ let _ = control_tx.try_send((KeyframeRequestReasonV3::Join, None));388+ }389+ Some(control_tx)390+ } else {391+ *state.media_control.lock().await = None;392+ None393+ };394+ let frame_events_in_flight = Arc::new(AtomicUsize::new(0));395+ *state.frame_delivery.lock().await =396+ Some((media_generation, Arc::clone(&frame_events_in_flight)));397+398+ // Input forwarder: absolute motion is latest-value state and may be399+ // dropped at the 60 Hz boundary. Relative motion is displacement, so it400+ // owns a separate accumulator and timer that coalesces rather than drops.401+ let input_send_diagnostics = Arc::clone(&network_diagnostics);402+ tokio::spawn(run_input_forwarder(403+ input_send,404+ rx,405+ use_v1,406+ input_capabilities,407+ input_send_diagnostics,408+ ));409+410+ // Frame reader — dual path: WebCodecs (raw bytes) or software JPEG decode411+ let use_webcodecs = state.webcodecs.load(Ordering::SeqCst);412+ tokio::spawn(run_video_delivery(VideoDeliveryRequest {413+ app,414+ endpoint,415+ frame_recv,416+ frame_connection: frame_connection_for_stats,417+ input_connection,418+ audio_connection: audio_connection_for_stats,419+ network_diagnostics,420+ media_control_requests,421+ upstream_moq_media,422+ frame_events_in_flight,423+ frame_channel,424+ media_transport,425+ media_generation,426+ connected_audio_generation,427+ use_webcodecs,428+ }));429+430+ let result = ConnectResult {431+ connected: true,432+ host_node_id: Some(host_node_id.to_string()),433+ development_mode,434+ media_transport: media_transport.diagnostic_name(),435+ pointer_surface_dimensions,436+ relative_pointer_available: input_availability.relative_pointer,437+ pointer_position_feedback_available: input_availability.pointer_position_feedback,438+ absolute_pointer_available: input_availability.absolute_pointer,439+ keyboard_available: input_availability.keyboard,440+ text_available: input_availability.text,441+ gamepad_available: input_availability.gamepad,442+ control_available: input_availability.control,443+ audio_available,444+ audio_generation: connected_audio_generation,445+ audio_error,446+ media_generation,447+ adaptive_feedback_available,448+ adaptive_feedback_error,449+ error: None,450+ };451+ connect_guard.commit();452+ Ok(result)453+}454+455+pub(crate) async fn disconnect_client(state: &AppState) -> Result<bool, String> {456+ let _connection_serial = state.client_connection_serial.lock().await;457+ next_audio_generation(&state.audio_connection_generation)?;458+ lock_audio_deliveries(&state.audio_deliveries).clear();459+ *state.media_control.lock().await = None;460+ if let Some((_generation, connection, _sender)) = state.media_feedback.lock().await.take() {461+ connection.close(0_u32.into(), b"client disconnected");462+ }463+ // Do not rely on endpoint shutdown alone to retire the session. The frame464+ // reader and diagnostics task both own connection clones, and a surviving465+ // media connection keeps the host's encoder and one-client lease alive.466+ // Closing the media connection explicitly gives the host an immediate,467+ // protocol-level session boundary.468+ close_generation_connection(state.media_connection.lock().await.take(), |connection| {469+ connection.close(0_u32.into(), b"client disconnected");470+ });471+ close_generation_connection(state.audio_connection.lock().await.take(), |connection| {472+ connection.close(0_u32.into(), b"client disconnected");473+ });474+ {475+ let mut ce = state.client_endpoint.lock().await;476+ if let Some(endpoint) = ce.take()477+ && tokio::time::timeout(CLIENT_ENDPOINT_CLOSE_TIMEOUT, endpoint.close())478+ .await479+ .is_err()480+ {481+ eprintln!(482+ "[client] timed out waiting for endpoint shutdown after explicit connection close"483+ );484+ }485+ }486+ {487+ let mut input_send = state.input_send.lock().await;488+ *input_send = None;489+ }490+ *state.frame_delivery.lock().await = None;491+ state492+ .client_connection_active493+ .store(false, Ordering::SeqCst);494+ Ok(true)495+}496+497+498+mod tests {499+ use super::*;500+501+ 502+ fn client_connect_guard_rejects_overlap_and_resets_only_failed_attempts() {503+ let active = Arc::new(AtomicBool::new(false));504+ let attempt = ClientConnectGuard::acquire(Arc::clone(&active)).unwrap();505+ assert!(ClientConnectGuard::acquire(Arc::clone(&active)).is_err());506+ drop(attempt);507+ assert!(!active.load(Ordering::SeqCst));508+509+ ClientConnectGuard::acquire(Arc::clone(&active))510+ .unwrap()511+ .commit();512+ assert!(active.load(Ordering::SeqCst));513+ assert!(ClientConnectGuard::acquire(Arc::clone(&active)).is_err());514+ active.store(false, Ordering::SeqCst);515+ }516+}@@ -0,0 +1,514 @@
1+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};2+3+use serde::Serialize;4+use sigil_protocol::{MAX_MEDIA_PAYLOAD_LEN, MAX_VIDEO_DIMENSION, MAX_VIDEO_PIXELS};5+use tauri::{AppHandle, Emitter};6+7+use crate::commands::state::AppState;8+9+pub(crate) fn byte_to_codec(value: u8) -> &'static str {10+ match value {11+ 1 => "h264",12+ 2 => "h265",13+ 3 => "av1",14+ _ => "h264",15+ }16+}17+18+19+pub struct FramePayload {20+ pub generation: u64,21+ pub width: u32,22+ pub height: u32,23+ pub data: String,24+ pub keyframe: bool,25+ pub codec: String,26+ pub capture_timestamp_micros: Option<u64>,27+ pub pts_micros: Option<i64>,28+ pub discontinuity: bool,29+}30+31+32+struct FrameErrorPayload {33+ generation: u64,34+ error: String,35+}36+37+pub(crate) fn emit_frame_error(app: &AppHandle, generation: u64, error: impl Into<String>) {38+ let _ = app.emit(39+ "frame-error",40+ FrameErrorPayload {41+ generation,42+ error: error.into(),43+ },44+ );45+}46+47+// Absorb brief webview→Rust acknowledgment jitter without allowing Tauri IPC48+// to grow without bound. Four 60 fps frames cap this handoff at about 67 ms;49+// WebCodecs has a separate, stricter decode-queue bound in the frontend.50+pub(crate) const CLIENT_FRAME_CHANNEL_CAPACITY: usize = 4;51+52+const FRAME_CHANNEL_MAGIC: [u8; 4] = *b"SGFR";53+const FRAME_CHANNEL_VERSION: u8 = 1;54+const FRAME_CHANNEL_HEADER_LEN: usize = 40;55+const FRAME_CHANNEL_FLAG_KEYFRAME: u8 = 1 << 0;56+const FRAME_CHANNEL_FLAG_DISCONTINUITY: u8 = 1 << 1;57+const FRAME_CHANNEL_FLAG_CODEC_CONFIG: u8 = 1 << 2;58+const FRAME_CHANNEL_OPTIONAL_U64_NONE: u64 = u64::MAX;59+const FRAME_CHANNEL_OPTIONAL_I64_NONE: i64 = i64::MIN;60+61+pub(crate) fn close_generation_connection<T>(62+ connection: Option<(u64, T)>,63+ close: impl FnOnce(T),64+) -> Option<u64> {65+ connection.map(|(generation, connection)| {66+ close(connection);67+ generation68+ })69+}70+71+pub(crate) fn take_generation_owned<T>(72+ slot: &mut Option<(u64, T)>,73+ expected_generation: u64,74+) -> Option<T> {75+ if slot76+ .as_ref()77+ .is_some_and(|(generation, _)| *generation == expected_generation)78+ {79+ slot.take().map(|(_, value)| value)80+ } else {81+ None82+ }83+}84+85+pub(crate) fn take_generation_owned_triple<T, U>(86+ slot: &mut Option<(u64, T, U)>,87+ expected_generation: u64,88+) -> Option<(T, U)> {89+ if slot90+ .as_ref()91+ .is_some_and(|(generation, _, _)| *generation == expected_generation)92+ {93+ slot.take().map(|(_, value, companion)| (value, companion))94+ } else {95+ None96+ }97+}98+99+pub(crate) struct FrameEnvelopeMetadata<'a> {100+ pub(crate) width: u32,101+ pub(crate) height: u32,102+ pub(crate) codec: &'a str,103+ pub(crate) keyframe: bool,104+ pub(crate) discontinuity: bool,105+ pub(crate) codec_config: bool,106+ pub(crate) sequence: Option<u64>,107+ pub(crate) capture_timestamp_micros: Option<u64>,108+ pub(crate) pts_micros: Option<i64>,109+}110+111+pub(crate) fn encode_frame_envelope(112+ metadata: FrameEnvelopeMetadata<'_>,113+ payload: &[u8],114+) -> Result<Vec<u8>, String> {115+ validate_legacy_media_header(metadata.width, metadata.height, payload.len())?;116+ if metadata.codec_config && !metadata.keyframe {117+ return Err("Frame codec configuration requires a keyframe".to_string());118+ }119+ if metadata.sequence == Some(FRAME_CHANNEL_OPTIONAL_U64_NONE) {120+ return Err("Frame sequence collides with the channel sentinel".to_string());121+ }122+ if metadata.capture_timestamp_micros == Some(FRAME_CHANNEL_OPTIONAL_U64_NONE) {123+ return Err("Capture timestamp collides with the channel sentinel".to_string());124+ }125+ if metadata.pts_micros == Some(FRAME_CHANNEL_OPTIONAL_I64_NONE) {126+ return Err("Frame PTS collides with the channel sentinel".to_string());127+ }128+ let width = u16::try_from(metadata.width).map_err(|_| {129+ format!(130+ "Frame width does not fit channel envelope: {}",131+ metadata.width132+ )133+ })?;134+ let height = u16::try_from(metadata.height).map_err(|_| {135+ format!(136+ "Frame height does not fit channel envelope: {}",137+ metadata.height138+ )139+ })?;140+ let payload_len = u32::try_from(payload.len()).map_err(|_| {141+ format!(142+ "Frame payload does not fit channel envelope: {}",143+ payload.len()144+ )145+ })?;146+ let codec = match metadata.codec {147+ "h264" => 1,148+ "h265" => 2,149+ "av1" => 3,150+ other => return Err(format!("Unsupported frame channel codec: {other}")),151+ };152+ let mut flags = 0_u8;153+ if metadata.keyframe {154+ flags |= FRAME_CHANNEL_FLAG_KEYFRAME;155+ }156+ if metadata.discontinuity {157+ flags |= FRAME_CHANNEL_FLAG_DISCONTINUITY;158+ }159+ if metadata.codec_config {160+ flags |= FRAME_CHANNEL_FLAG_CODEC_CONFIG;161+ }162+163+ let mut envelope = Vec::with_capacity(FRAME_CHANNEL_HEADER_LEN + payload.len());164+ envelope.extend_from_slice(&FRAME_CHANNEL_MAGIC);165+ envelope.push(FRAME_CHANNEL_VERSION);166+ envelope.push(codec);167+ envelope.push(flags);168+ envelope.push(0); // Reserved; the parser rejects non-zero values.169+ envelope.extend_from_slice(&width.to_be_bytes());170+ envelope.extend_from_slice(&height.to_be_bytes());171+ envelope.extend_from_slice(&payload_len.to_be_bytes());172+ envelope.extend_from_slice(173+ &metadata174+ .sequence175+ .unwrap_or(FRAME_CHANNEL_OPTIONAL_U64_NONE)176+ .to_be_bytes(),177+ );178+ envelope.extend_from_slice(179+ &metadata180+ .capture_timestamp_micros181+ .unwrap_or(FRAME_CHANNEL_OPTIONAL_U64_NONE)182+ .to_be_bytes(),183+ );184+ envelope.extend_from_slice(185+ &metadata186+ .pts_micros187+ .unwrap_or(FRAME_CHANNEL_OPTIONAL_I64_NONE)188+ .to_be_bytes(),189+ );190+ envelope.extend_from_slice(payload);191+ debug_assert_eq!(envelope.len(), FRAME_CHANNEL_HEADER_LEN + payload.len());192+ Ok(envelope)193+}194+195+pub(crate) fn try_reserve_frame_channel_slot(in_flight: &AtomicUsize) -> bool {196+ in_flight197+ .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |current| {198+ (current < CLIENT_FRAME_CHANNEL_CAPACITY).then_some(current + 1)199+ })200+ .is_ok()201+}202+203+pub(crate) fn release_frame_channel_slot(in_flight: &AtomicUsize) {204+ let _ = in_flight.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |current| {205+ current.checked_sub(1)206+ });207+}208+209+fn release_frame_channel_slot_for_generation(210+ in_flight: &AtomicUsize,211+ current_generation: u64,212+ generation: u64,213+) -> bool {214+ if generation == 0 || current_generation != generation {215+ return false;216+ }217+ release_frame_channel_slot(in_flight);218+ true219+}220+221+pub(crate) async fn acknowledge_frame_delivery(222+ state: &AppState,223+ generation: u64,224+) -> Result<bool, String> {225+ // Serialize selection of the generation-owned counter against connect and226+ // disconnect. Each media task keeps its own counter, so an old callback can227+ // never consume a permit reserved by a replacement session.228+ let _connection_serial = state.client_connection_serial.lock().await;229+ let delivery = state.frame_delivery.lock().await;230+ let Some((current_generation, in_flight)) = delivery.as_ref() else {231+ return Ok(false);232+ };233+ Ok(release_frame_channel_slot_for_generation(234+ in_flight,235+ *current_generation,236+ generation,237+ ))238+}239+240+pub(crate) fn next_media_generation(counter: &AtomicU64) -> Result<u64, String> {241+ counter242+ .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |current| {243+ current.checked_add(1)244+ })245+ .map(|previous| previous + 1)246+ .map_err(|_| "Media connection generation overflowed".to_string())247+}248+249+pub(crate) fn validate_legacy_media_header(250+ width: u32,251+ height: u32,252+ payload_len: usize,253+) -> Result<(), String> {254+ if width == 0255+ || height == 0256+ || width > u32::from(MAX_VIDEO_DIMENSION)257+ || height > u32::from(MAX_VIDEO_DIMENSION)258+ || width.saturating_mul(height) > MAX_VIDEO_PIXELS259+ {260+ return Err(format!("Invalid legacy media dimensions: {width}x{height}"));261+ }262+ if payload_len == 0 || payload_len > MAX_MEDIA_PAYLOAD_LEN {263+ return Err(format!(264+ "Invalid legacy media payload length: {payload_len}"265+ ));266+ }267+ Ok(())268+}269+270+271+mod tests {272+ use super::*;273+274+ 275+ fn media_generations_are_nonzero_monotonic_and_checked_for_overflow() {276+ let counter = AtomicU64::new(0);277+ assert_eq!(next_media_generation(&counter).unwrap(), 1);278+ assert_eq!(next_media_generation(&counter).unwrap(), 2);279+280+ let exhausted = AtomicU64::new(u64::MAX);281+ assert!(next_media_generation(&exhausted).is_err());282+ }283+284+ 285+ fn binary_frame_envelope_has_exact_stable_layout() {286+ let payload = [0, 0, 0, 1, 0x65];287+ let envelope = encode_frame_envelope(288+ FrameEnvelopeMetadata {289+ width: 1280,290+ height: 800,291+ codec: "h264",292+ keyframe: true,293+ discontinuity: true,294+ codec_config: true,295+ sequence: Some(42),296+ capture_timestamp_micros: Some(123_456),297+ pts_micros: Some(98_765),298+ },299+ &payload,300+ )301+ .unwrap();302+303+ assert_eq!(envelope.len(), FRAME_CHANNEL_HEADER_LEN + payload.len());304+ assert_eq!(&envelope[0..4], b"SGFR");305+ assert_eq!(envelope[4], 1);306+ assert_eq!(envelope[5], 1);307+ assert_eq!(envelope[6], 0b111);308+ assert_eq!(envelope[7], 0);309+ assert_eq!(&envelope[8..10], &1280_u16.to_be_bytes());310+ assert_eq!(&envelope[10..12], &800_u16.to_be_bytes());311+ assert_eq!(&envelope[12..16], &5_u32.to_be_bytes());312+ assert_eq!(&envelope[16..24], &42_u64.to_be_bytes());313+ assert_eq!(&envelope[24..32], &123_456_u64.to_be_bytes());314+ assert_eq!(&envelope[32..40], &98_765_i64.to_be_bytes());315+ assert_eq!(&envelope[40..], payload);316+ }317+318+ 319+ fn binary_frame_envelope_uses_explicit_optional_sentinels() {320+ let envelope = encode_frame_envelope(321+ FrameEnvelopeMetadata {322+ width: 1,323+ height: 1,324+ codec: "av1",325+ keyframe: false,326+ discontinuity: false,327+ codec_config: false,328+ sequence: None,329+ capture_timestamp_micros: None,330+ pts_micros: None,331+ },332+ &[1],333+ )334+ .unwrap();335+ assert_eq!(envelope[5], 3);336+ assert_eq!(&envelope[16..24], &u64::MAX.to_be_bytes());337+ assert_eq!(&envelope[24..32], &u64::MAX.to_be_bytes());338+ assert_eq!(&envelope[32..40], &i64::MIN.to_be_bytes());339+ }340+341+ 342+ fn binary_frame_envelope_rejects_invalid_metadata_before_sending() {343+ let metadata = |codec| FrameEnvelopeMetadata {344+ width: 1280,345+ height: 800,346+ codec,347+ keyframe: false,348+ discontinuity: false,349+ codec_config: false,350+ sequence: None,351+ capture_timestamp_micros: None,352+ pts_micros: None,353+ };354+ assert!(encode_frame_envelope(metadata("vp9"), &[1]).is_err());355+ assert!(encode_frame_envelope(metadata("h264"), &[]).is_err());356+ assert!(357+ encode_frame_envelope(358+ FrameEnvelopeMetadata {359+ codec_config: true,360+ ..metadata("h264")361+ },362+ &[1]363+ )364+ .is_err()365+ );366+ assert!(367+ encode_frame_envelope(368+ FrameEnvelopeMetadata {369+ sequence: Some(u64::MAX),370+ ..metadata("h264")371+ },372+ &[1]373+ )374+ .is_err()375+ );376+ assert!(377+ encode_frame_envelope(378+ FrameEnvelopeMetadata {379+ pts_micros: Some(i64::MIN),380+ ..metadata("h264")381+ },382+ &[1]383+ )384+ .is_err()385+ );386+ assert!(387+ encode_frame_envelope(388+ FrameEnvelopeMetadata {389+ width: u32::from(MAX_VIDEO_DIMENSION) + 1,390+ ..metadata("h264")391+ },392+ &[1]393+ )394+ .is_err()395+ );396+ }397+398+ 399+ fn frame_channel_slots_are_bounded_and_cannot_underflow() {400+ let in_flight = AtomicUsize::new(0);401+ release_frame_channel_slot(&in_flight);402+ assert_eq!(in_flight.load(Ordering::SeqCst), 0);403+404+ for expected in 1..=CLIENT_FRAME_CHANNEL_CAPACITY {405+ assert!(try_reserve_frame_channel_slot(&in_flight));406+ assert_eq!(in_flight.load(Ordering::SeqCst), expected);407+ }408+ assert!(!try_reserve_frame_channel_slot(&in_flight));409+ assert_eq!(410+ in_flight.load(Ordering::SeqCst),411+ CLIENT_FRAME_CHANNEL_CAPACITY412+ );413+414+ for expected in (0..CLIENT_FRAME_CHANNEL_CAPACITY).rev() {415+ release_frame_channel_slot(&in_flight);416+ assert_eq!(in_flight.load(Ordering::SeqCst), expected);417+ }418+ release_frame_channel_slot(&in_flight);419+ assert_eq!(in_flight.load(Ordering::SeqCst), 0);420+ }421+422+ 423+ fn frame_acknowledgments_release_only_the_matching_generation() {424+ let in_flight = AtomicUsize::new(2);425+ let generation = 9;426+427+ assert!(!release_frame_channel_slot_for_generation(428+ &in_flight, generation, 8429+ ));430+ assert!(!release_frame_channel_slot_for_generation(431+ &in_flight, generation, 0432+ ));433+ assert_eq!(in_flight.load(Ordering::SeqCst), 2);434+ assert!(release_frame_channel_slot_for_generation(435+ &in_flight, generation, 9436+ ));437+ assert_eq!(in_flight.load(Ordering::SeqCst), 1);438+ }439+440+ 441+ fn global_frame_events_serialize_their_media_generation() {442+ let frame = serde_json::to_value(FramePayload {443+ generation: 17,444+ width: 1280,445+ height: 800,446+ data: "jpeg".to_string(),447+ keyframe: true,448+ codec: "h264".to_string(),449+ capture_timestamp_micros: Some(1),450+ pts_micros: Some(2),451+ discontinuity: false,452+ })453+ .unwrap();454+ let error = serde_json::to_value(FrameErrorPayload {455+ generation: 17,456+ error: "closed".to_string(),457+ })458+ .unwrap();459+460+ assert_eq!(frame["generation"], 17);461+ assert_eq!(error["generation"], 17);462+ assert_eq!(error["error"], "closed");463+ }464+465+ 466+ fn generation_connection_close_retires_exactly_one_owned_handle() {467+ let closes = AtomicUsize::new(0);468+ let retired = close_generation_connection(Some((7, "media")), |connection| {469+ assert_eq!(connection, "media");470+ closes.fetch_add(1, Ordering::SeqCst);471+ });472+ assert_eq!(retired, Some(7));473+ assert_eq!(closes.load(Ordering::SeqCst), 1);474+475+ let absent: Option<(u64, &str)> = None;476+ assert_eq!(477+ close_generation_connection(absent, |_| {478+ closes.fetch_add(1, Ordering::SeqCst);479+ }),480+ None481+ );482+ assert_eq!(closes.load(Ordering::SeqCst), 1);483+ }484+485+ 486+ fn generation_owned_retirement_cannot_take_a_replacement_session() {487+ let mut slot = Some((12, "replacement"));488+ assert_eq!(take_generation_owned(&mut slot, 11), None);489+ assert_eq!(slot, Some((12, "replacement")));490+ assert_eq!(take_generation_owned(&mut slot, 12), Some("replacement"));491+ assert_eq!(slot, None);492+493+ let mut feedback = Some((22, "replacement connection", "replacement sender"));494+ assert_eq!(take_generation_owned_triple(&mut feedback, 21), None);495+ assert_eq!(496+ feedback,497+ Some((22, "replacement connection", "replacement sender"))498+ );499+ assert_eq!(500+ take_generation_owned_triple(&mut feedback, 22),501+ Some(("replacement connection", "replacement sender"))502+ );503+ assert_eq!(feedback, None);504+ }505+506+ 507+ fn legacy_media_header_is_bounded_before_allocation() {508+ assert!(validate_legacy_media_header(1280, 800, 1024).is_ok());509+ assert!(validate_legacy_media_header(0, 800, 1024).is_err());510+ assert!(validate_legacy_media_header(1280, 800, 0).is_err());511+ assert!(validate_legacy_media_header(1280, 800, MAX_MEDIA_PAYLOAD_LEN + 1).is_err());512+ assert!(validate_legacy_media_header(u32::MAX, u32::MAX, 1024).is_err());513+ }514+}@@ -0,0 +1,919 @@
1+use std::sync::{Arc, Mutex as StdMutex};2+use std::time::{Duration, Instant};3+4+use iroh::Endpoint;5+use serde::Serialize;6+use sigil_protocol::{7+ Capability, INPUT_ALPN_V1, InputEvent, InvitationGrants, PointerPosition,8+ RELATIVE_POINTER_DELTA_MAX, RELATIVE_POINTER_DELTA_MIN, read_input_ack, write_input_event,9+};10+use tauri::ipc::Channel;11+12+use crate::commands::state::AppState;13+use crate::media::network_diagnostics::{NetworkSessionDiagnostics, lock_network_diagnostics};14+use crate::media::transport::{NegotiatedV1Stream, negotiate_v1};15+16+const INPUT_ALPN: &[u8] = sigil_protocol::LEGACY_INPUT_ALPN_V0;17+pub(crate) const CLIENT_INPUT_QUEUE_CAPACITY: usize = 256;18+19+20+pub(crate) struct InputAvailability {21+ pub(crate) relative_pointer: bool,22+ pub(crate) pointer_position_feedback: bool,23+ pub(crate) absolute_pointer: bool,24+ pub(crate) keyboard: bool,25+ pub(crate) text: bool,26+ pub(crate) gamepad: bool,27+ pub(crate) input_ack: bool,28+ pub(crate) control: bool,29+}30+31+impl InputAvailability {32+ fn from_capabilities(capabilities: &[Capability]) -> Self {33+ let relative_pointer = capabilities.contains(&Capability::RelativePointer);34+ let pointer_position_feedback = capabilities.contains(&Capability::PointerPositionFeedback);35+ let absolute_pointer = capabilities.contains(&Capability::AbsolutePointer);36+ let keyboard = capabilities.contains(&Capability::Keyboard);37+ let text = capabilities.contains(&Capability::Text);38+ let gamepad = capabilities.contains(&Capability::Gamepad);39+ let input_ack = capabilities.contains(&Capability::InputAck);40+ Self {41+ relative_pointer,42+ pointer_position_feedback,43+ absolute_pointer,44+ keyboard,45+ text,46+ gamepad,47+ input_ack,48+ control: relative_pointer || absolute_pointer || keyboard || text || gamepad,49+ }50+ }51+}52+53+54+55+pub enum PointerFeedbackPayload {56+ Position {57+ sequence: u64,58+ position: Option<PointerPosition>,59+ 60+ pointer_visible: Option<bool>,61+ },62+ Terminal {63+ reason: PointerFeedbackTerminalReason,64+ },65+}66+67+68+69+pub enum PointerFeedbackTerminalReason {70+ Eof,71+ Malformed,72+}73+74+75+struct RelativePointerAccumulator {76+ dx: i64,77+ dy: i64,78+}79+80+impl RelativePointerAccumulator {81+ fn push(&mut self, dx: i32, dy: i32) {82+ self.dx = self.dx.saturating_add(i64::from(dx));83+ self.dy = self.dy.saturating_add(i64::from(dy));84+ }85+86+ fn take(&mut self) -> Option<InputEvent> {87+ if self.dx == 0 && self.dy == 0 {88+ return None;89+ }90+ let dx = self.dx.clamp(91+ i64::from(RELATIVE_POINTER_DELTA_MIN),92+ i64::from(RELATIVE_POINTER_DELTA_MAX),93+ ) as i32;94+ let dy = self.dy.clamp(95+ i64::from(RELATIVE_POINTER_DELTA_MIN),96+ i64::from(RELATIVE_POINTER_DELTA_MAX),97+ ) as i32;98+ let event = InputEvent::MouseMoveRelative { dx, dy };99+ self.dx -= i64::from(dx);100+ self.dy -= i64::from(dy);101+ Some(event)102+ }103+104+ fn is_pending(&self) -> bool {105+ self.dx != 0 || self.dy != 0106+ }107+}108+109+fn stage_relative_input(110+ pending: &mut RelativePointerAccumulator,111+ event: InputEvent,112+) -> Option<InputEvent> {113+ match event {114+ InputEvent::MouseMoveRelative { dx, dy } => {115+ pending.push(dx, dy);116+ None117+ }118+ event => Some(event),119+ }120+}121+122+async fn open_negotiated_input_stream(123+ endpoint: &Endpoint,124+ address: &iroh::EndpointAddr,125+ nonce: [u8; 16],126+ capabilities: Vec<Capability>,127+) -> Result<128+ (129+ iroh::endpoint::Connection,130+ iroh::endpoint::SendStream,131+ iroh::endpoint::RecvStream,132+ NegotiatedV1Stream,133+ ),134+ String,135+> {136+ let connection = endpoint137+ .connect(address.clone(), INPUT_ALPN_V1)138+ .await139+ .map_err(|error| format!("Failed to connect input stream: {error}"))?;140+ let (mut send, mut recv) = connection141+ .open_bi()142+ .await143+ .map_err(|error| format!("Failed to open input stream: {error}"))?;144+ let negotiation = negotiate_v1(145+ &mut send,146+ &mut recv,147+ nonce,148+ capabilities,149+ None,150+ "input",151+ None,152+ )153+ .await?;154+ Ok((connection, send, recv, negotiation))155+}156+157+fn input_capability_offers(grants: InvitationGrants) -> Vec<Vec<Capability>> {158+ let base = [159+ vec![160+ Capability::RelativePointer,161+ Capability::PointerPositionFeedback,162+ Capability::PointerVisibilityFeedback,163+ Capability::AbsolutePointer,164+ Capability::Keyboard,165+ Capability::Text,166+ Capability::Gamepad,167+ ],168+ vec![169+ Capability::RelativePointer,170+ Capability::PointerPositionFeedback,171+ Capability::AbsolutePointer,172+ Capability::Keyboard,173+ Capability::Text,174+ Capability::Gamepad,175+ ],176+ vec![177+ Capability::RelativePointer,178+ Capability::AbsolutePointer,179+ Capability::Keyboard,180+ Capability::Text,181+ Capability::Gamepad,182+ ],183+ vec![184+ Capability::AbsolutePointer,185+ Capability::Keyboard,186+ Capability::Text,187+ Capability::Gamepad,188+ ],189+ ];190+ let has_input_grant = grants.contains(InvitationGrants::POINTER_KEYBOARD)191+ || grants.contains(InvitationGrants::GAMEPAD);192+ let mut offers = Vec::with_capacity(base.len() * 2);193+ if has_input_grant {194+ for mut offer in base.clone() {195+ offer.push(Capability::InputAck);196+ offers.push(offer);197+ }198+ }199+ offers.extend(base);200+ for offer in &mut offers {201+ offer.retain(|capability| match capability {202+ Capability::Gamepad => grants.contains(InvitationGrants::GAMEPAD),203+ Capability::AbsolutePointer204+ | Capability::RelativePointer205+ | Capability::Keyboard206+ | Capability::Text207+ | Capability::PointerPositionFeedback208+ | Capability::PointerVisibilityFeedback => {209+ grants.contains(InvitationGrants::POINTER_KEYBOARD)210+ }211+ Capability::InputAck => has_input_grant,212+ _ => true,213+ });214+ }215+ offers.dedup();216+ offers217+}218+219+fn input_event_allowed(capabilities: &[Capability], event: &InputEvent) -> bool {220+ match event {221+ InputEvent::Probe => capabilities.contains(&Capability::InputAck),222+ InputEvent::MouseMove { .. } => capabilities.contains(&Capability::AbsolutePointer),223+ InputEvent::MouseMoveRelative { .. } => capabilities.contains(&Capability::RelativePointer),224+ InputEvent::MousePositionSync { .. } => capabilities.contains(&Capability::RelativePointer),225+ InputEvent::MouseClick { .. }226+ | InputEvent::MouseDown { .. }227+ | InputEvent::MouseUp { .. }228+ | InputEvent::MouseScroll { .. } => {229+ capabilities.contains(&Capability::RelativePointer)230+ || capabilities.contains(&Capability::AbsolutePointer)231+ }232+ InputEvent::KeyDown { .. } | InputEvent::KeyUp { .. } | InputEvent::KeyClick { .. } => {233+ capabilities.contains(&Capability::Keyboard)234+ }235+ InputEvent::Text { .. } => capabilities.contains(&Capability::Text),236+ InputEvent::Gamepad { .. } => capabilities.contains(&Capability::Gamepad),237+ }238+}239+240+async fn write_client_input_event(241+ stream: &mut iroh::endpoint::SendStream,242+ event: &InputEvent,243+ use_v1: bool,244+ diagnostics: Option<&Arc<StdMutex<NetworkSessionDiagnostics>>>,245+) -> Result<(), String> {246+ if let Some(diagnostics) = diagnostics {247+ lock_network_diagnostics(diagnostics).begin_input_send(Instant::now());248+ }249+ if use_v1 {250+ write_input_event(stream, event)251+ .await252+ .map_err(|error| error.to_string())253+ } else {254+ let json = serde_json::to_string(event).map_err(|error| error.to_string())?;255+ stream256+ .write_all(format!("{json}\n").as_bytes())257+ .await258+ .map_err(|error| error.to_string())259+ }260+}261+262+fn observe_input_ack_if_negotiated(263+ diagnostics: &StdMutex<NetworkSessionDiagnostics>,264+ negotiated: bool,265+ sequence: u64,266+ now: Instant,267+) -> Result<(), String> {268+ if !negotiated {269+ return Ok(());270+ }271+ lock_network_diagnostics(diagnostics).observe_input_ack(sequence, now)272+}273+274+pub(crate) struct InputSession {275+ pub(crate) connection: iroh::endpoint::Connection,276+ pub(crate) send: iroh::endpoint::SendStream,277+ pub(crate) recv: iroh::endpoint::RecvStream,278+ pub(crate) capabilities: Vec<Capability>,279+ pub(crate) availability: InputAvailability,280+}281+282+pub(crate) async fn open_input_session(283+ endpoint: &Endpoint,284+ address: &iroh::EndpointAddr,285+ nonce: [u8; 16],286+ media_session_id: Option<u64>,287+ grants: InvitationGrants,288+ use_v1: bool,289+) -> Result<InputSession, String> {290+ let input_alpn = if use_v1 { INPUT_ALPN_V1 } else { INPUT_ALPN };291+ let (connection, send, recv, capabilities) = if use_v1 {292+ let mut errors = Vec::new();293+ let mut accepted = None;294+ // Older hosts reject unknown capability enum values. Try all four295+ // pointer feature levels with ACK first, then repeat the exact legacy296+ // offers without ACK so lack of diagnostics never forces absolute297+ // pointer input prematurely.298+ for capabilities in input_capability_offers(grants) {299+ match open_negotiated_input_stream(endpoint, address, nonce, capabilities).await {300+ Ok(result) => {301+ accepted = Some(result);302+ break;303+ }304+ Err(error) => errors.push(error),305+ }306+ }307+ let (connection, send, recv, input_negotiation) = accepted308+ .ok_or_else(|| format!("All input capability offers failed: {}", errors.join("; ")))?;309+ if Some(input_negotiation.session_id) != media_session_id {310+ return Err("Host returned mismatched media and input sessions".to_string());311+ }312+ (connection, send, recv, input_negotiation.capabilities)313+ } else {314+ let input_conn = endpoint315+ .connect(address.clone(), input_alpn)316+ .await317+ .map_err(|e| format!("Failed to connect input stream: {}", e))?;318+ let (mut send, recv) = input_conn319+ .open_bi()320+ .await321+ .map_err(|e| format!("Failed to open input stream: {}", e))?;322+ send.write_all(&[1u8])323+ .await324+ .map_err(|e| format!("Failed to send input start: {}", e))?;325+ // The inherited protocol predates negotiation and supports the current326+ // absolute-pointer, keyboard, and text event set.327+ (328+ input_conn,329+ send,330+ recv,331+ vec![332+ Capability::AbsolutePointer,333+ Capability::Keyboard,334+ Capability::Text,335+ ],336+ )337+ };338+ let availability = InputAvailability::from_capabilities(&capabilities);339+ Ok(InputSession {340+ connection,341+ send,342+ recv,343+ capabilities,344+ availability,345+ })346+}347+348+pub(crate) async fn run_input_feedback(349+ mut input_feedback: iroh::endpoint::RecvStream,350+ feedback_diagnostics: Arc<StdMutex<NetworkSessionDiagnostics>>,351+ mut pointer_feedback_enabled: bool,352+ input_ack_enabled: bool,353+ pointer_channel: Channel<PointerFeedbackPayload>,354+) {355+ let terminal_reason = loop {356+ let response = match read_input_ack(&mut input_feedback).await {357+ Ok(Some(response)) => response,358+ Ok(None) => {359+ lock_network_diagnostics(&feedback_diagnostics).mark_input_feedback_closed();360+ break PointerFeedbackTerminalReason::Eof;361+ }362+ Err(error) => {363+ lock_network_diagnostics(&feedback_diagnostics).mark_input_feedback_malformed();364+ eprintln!("[client] invalid input feedback: {error}");365+ break PointerFeedbackTerminalReason::Malformed;366+ }367+ };368+ if let Err(error) = observe_input_ack_if_negotiated(369+ &feedback_diagnostics,370+ input_ack_enabled,371+ response.sequence,372+ Instant::now(),373+ ) {374+ eprintln!("[client] invalid input acknowledgement: {error}");375+ break PointerFeedbackTerminalReason::Malformed;376+ }377+ if pointer_feedback_enabled378+ && pointer_channel379+ .send(PointerFeedbackPayload::Position {380+ sequence: response.sequence,381+ position: response.pointer_position,382+ pointer_visible: response.pointer_visible,383+ })384+ .is_err()385+ {386+ // Losing the webview's pointer channel must not stop ACK387+ // draining and apply backpressure to host input.388+ pointer_feedback_enabled = false;389+ if !input_ack_enabled {390+ return;391+ }392+ }393+ };394+ // The session-owned channel emits at most one terminal message.395+ // JavaScript rejects deliveries from superseded channel closures.396+ let _ = pointer_channel.send(PointerFeedbackPayload::Terminal {397+ reason: terminal_reason,398+ });399+}400+401+pub(crate) async fn run_input_forwarder(402+ mut input_stream: iroh::endpoint::SendStream,403+ rx: tokio::sync::mpsc::Receiver<InputEvent>,404+ use_v1: bool,405+ input_capabilities: Vec<Capability>,406+ input_send_diagnostics: Arc<StdMutex<NetworkSessionDiagnostics>>,407+) {408+ let mut rx = rx;409+ const MOUSE_INTERVAL: Duration = Duration::from_millis(16);410+ let started = Instant::now();411+ let mut last_absolute_mouse_time = started.checked_sub(MOUSE_INTERVAL).unwrap_or(started);412+ let mut last_relative_mouse_time = started.checked_sub(MOUSE_INTERVAL).unwrap_or(started);413+ let mut pending_relative = RelativePointerAccumulator::default();414+ let mut input_open = true;415+416+ while input_open {417+ let event = if pending_relative.is_pending() {418+ let wait = MOUSE_INTERVAL.saturating_sub(last_relative_mouse_time.elapsed());419+ if wait.is_zero() {420+ let Some(event) = pending_relative.take() else {421+ continue;422+ };423+ if let Err(error) = write_client_input_event(424+ &mut input_stream,425+ &event,426+ use_v1,427+ Some(&input_send_diagnostics),428+ )429+ .await430+ {431+ eprintln!("[client] input stream write failed: {error}; disconnecting");432+ break;433+ }434+ last_relative_mouse_time = Instant::now();435+ continue;436+ }437+ tokio::select! {438+ event = rx.recv() => event,439+ () = tokio::time::sleep(wait) => {440+ let Some(event) = pending_relative.take() else {441+ continue;442+ };443+ if let Err(error) = write_client_input_event(444+ &mut input_stream,445+ &event,446+ use_v1,447+ Some(&input_send_diagnostics),448+ ).await {449+ eprintln!("[client] input stream write failed: {error}; disconnecting");450+ break;451+ }452+ last_relative_mouse_time = Instant::now();453+ continue;454+ }455+ }456+ } else {457+ rx.recv().await458+ };459+ let Some(event) = event else {460+ input_open = false;461+ continue;462+ };463+ // The host's accepted capability set is an authorization boundary.464+ // Drop unavailable event classes silently so event contents never465+ // reach logs or the wire even if a compromised webview invokes the466+ // command directly.467+ if !input_event_allowed(&input_capabilities, &event) {468+ continue;469+ }470+ let Some(event) = stage_relative_input(&mut pending_relative, event) else {471+ continue;472+ };473+ let mut flushed_relative_barrier = false;474+ while let Some(relative_barrier) = pending_relative.take() {475+ if let Err(error) = write_client_input_event(476+ &mut input_stream,477+ &relative_barrier,478+ use_v1,479+ Some(&input_send_diagnostics),480+ )481+ .await482+ {483+ eprintln!("[client] input stream write failed: {error}; disconnecting");484+ input_open = false;485+ break;486+ }487+ flushed_relative_barrier = true;488+ }489+ if !input_open {490+ break;491+ }492+ if flushed_relative_barrier {493+ last_relative_mouse_time = Instant::now();494+ }495+ if matches!(event, InputEvent::MouseMove { .. }) {496+ let now = Instant::now();497+ if now.duration_since(last_absolute_mouse_time) < MOUSE_INTERVAL {498+ continue;499+ }500+ last_absolute_mouse_time = now;501+ }502+ if let Err(error) = write_client_input_event(503+ &mut input_stream,504+ &event,505+ use_v1,506+ Some(&input_send_diagnostics),507+ )508+ .await509+ {510+ eprintln!("[client] input stream write failed: {error}; disconnecting");511+ break;512+ }513+ }514+ while let Some(event) = pending_relative.take() {515+ if let Err(error) = write_client_input_event(516+ &mut input_stream,517+ &event,518+ use_v1,519+ Some(&input_send_diagnostics),520+ )521+ .await522+ {523+ eprintln!("[client] final relative input write failed: {error}");524+ break;525+ }526+ }527+ let _ = input_stream.finish();528+}529+530+pub(crate) async fn send_input(state: &AppState, event: InputEvent) -> Result<bool, String> {531+ let tx = state532+ .input_send533+ .lock()534+ .await535+ .clone()536+ .ok_or_else(|| "Not connected to host".to_string())?;537+ match tx.try_send(event) {538+ Ok(()) => Ok(true),539+ Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => Ok(false),540+ Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => {541+ Err("Input channel closed".to_string())542+ }543+ }544+}545+546+547+mod tests {548+ use super::*;549+550+ 551+ fn pointer_feedback_channel_has_explicit_bounded_terminal_envelopes() {552+ let position = serde_json::to_value(PointerFeedbackPayload::Position {553+ sequence: 7,554+ position: Some(PointerPosition { x: 1280, y: 800 }),555+ pointer_visible: Some(false),556+ })557+ .unwrap();558+ assert_eq!(position["type"], "position");559+ assert_eq!(position["sequence"], 7);560+ assert_eq!(position["position"]["x"], 1280);561+ assert_eq!(position["pointer_visible"], false);562+563+ let legacy = serde_json::to_value(PointerFeedbackPayload::Position {564+ sequence: 8,565+ position: Some(PointerPosition { x: 640, y: 400 }),566+ pointer_visible: None,567+ })568+ .unwrap();569+ assert!(legacy.get("pointer_visible").is_none());570+571+ let eof = serde_json::to_value(PointerFeedbackPayload::Terminal {572+ reason: PointerFeedbackTerminalReason::Eof,573+ })574+ .unwrap();575+ assert_eq!(576+ eof,577+ serde_json::json!({ "type": "terminal", "reason": "eof" })578+ );579+580+ let malformed = serde_json::to_value(PointerFeedbackPayload::Terminal {581+ reason: PointerFeedbackTerminalReason::Malformed,582+ })583+ .unwrap();584+ assert_eq!(585+ malformed,586+ serde_json::json!({ "type": "terminal", "reason": "malformed" })587+ );588+ }589+590+ 591+ fn input_events_require_their_negotiated_capability() {592+ let pointer = [Capability::AbsolutePointer];593+ assert!(input_event_allowed(594+ &pointer,595+ &InputEvent::MouseMove { x: 1, y: 2 }596+ ));597+ assert!(input_event_allowed(598+ &pointer,599+ &InputEvent::MouseClick { b: 1 }600+ ));601+ assert!(input_event_allowed(602+ &pointer,603+ &InputEvent::MouseDown { b: 1 }604+ ));605+ assert!(input_event_allowed(&pointer, &InputEvent::MouseUp { b: 1 }));606+ assert!(input_event_allowed(607+ &pointer,608+ &InputEvent::MouseScroll { dx: 0, dy: 1 }609+ ));610+ assert!(!input_event_allowed(611+ &pointer,612+ &InputEvent::MouseMoveRelative { dx: 1, dy: -2 }613+ ));614+ assert!(!input_event_allowed(615+ &pointer,616+ &InputEvent::MousePositionSync { x: 640, y: 400 }617+ ));618+ assert!(!input_event_allowed(619+ &pointer,620+ &InputEvent::KeyDown { k: "A".into() }621+ ));622+ assert!(!input_event_allowed(623+ &pointer,624+ &InputEvent::Text { s: "a".into() }625+ ));626+627+ let relative_pointer = [Capability::RelativePointer];628+ assert!(input_event_allowed(629+ &relative_pointer,630+ &InputEvent::MouseMoveRelative { dx: 1, dy: -2 }631+ ));632+ assert!(input_event_allowed(633+ &relative_pointer,634+ &InputEvent::MousePositionSync { x: 640, y: 400 }635+ ));636+ assert!(input_event_allowed(637+ &relative_pointer,638+ &InputEvent::MouseDown { b: 1 }639+ ));640+ assert!(input_event_allowed(641+ &relative_pointer,642+ &InputEvent::MouseUp { b: 1 }643+ ));644+ assert!(input_event_allowed(645+ &relative_pointer,646+ &InputEvent::MouseScroll { dx: 0, dy: 1 }647+ ));648+ assert!(!input_event_allowed(649+ &relative_pointer,650+ &InputEvent::MouseMove { x: 1, y: 2 }651+ ));652+653+ let keyboard = [Capability::Keyboard];654+ assert!(input_event_allowed(655+ &keyboard,656+ &InputEvent::KeyDown { k: "A".into() }657+ ));658+ assert!(input_event_allowed(659+ &keyboard,660+ &InputEvent::KeyUp { k: "A".into() }661+ ));662+ assert!(input_event_allowed(663+ &keyboard,664+ &InputEvent::KeyClick { k: "A".into() }665+ ));666+ assert!(!input_event_allowed(667+ &keyboard,668+ &InputEvent::Text { s: "a".into() }669+ ));670+671+ let text = [Capability::Text];672+ assert!(input_event_allowed(673+ &text,674+ &InputEvent::Text { s: "a".into() }675+ ));676+ assert!(!input_event_allowed(677+ &text,678+ &InputEvent::KeyDown { k: "A".into() }679+ ));680+681+ let gamepad = [Capability::Gamepad];682+ assert!(input_event_allowed(683+ &gamepad,684+ &InputEvent::Gamepad {685+ state: sigil_protocol::GamepadState::default(),686+ }687+ ));688+ assert!(!input_event_allowed(689+ &keyboard,690+ &InputEvent::Gamepad {691+ state: sigil_protocol::GamepadState::default(),692+ }693+ ));694+ }695+696+ 697+ fn empty_input_capabilities_are_view_only() {698+ let capabilities = [];699+ assert!(!input_event_allowed(700+ &capabilities,701+ &InputEvent::MouseMove { x: 1, y: 2 }702+ ));703+ assert!(!input_event_allowed(704+ &capabilities,705+ &InputEvent::KeyDown { k: "A".into() }706+ ));707+ assert!(!input_event_allowed(708+ &capabilities,709+ &InputEvent::Text { s: "a".into() }710+ ));711+ assert_eq!(712+ InputAvailability::from_capabilities(&capabilities),713+ InputAvailability {714+ relative_pointer: false,715+ pointer_position_feedback: false,716+ absolute_pointer: false,717+ keyboard: false,718+ text: false,719+ gamepad: false,720+ input_ack: false,721+ control: false,722+ }723+ );724+ }725+726+ 727+ fn input_availability_reports_each_accepted_capability_exactly() {728+ assert_eq!(729+ InputAvailability::from_capabilities(&[Capability::AbsolutePointer, Capability::Text]),730+ InputAvailability {731+ relative_pointer: false,732+ pointer_position_feedback: false,733+ absolute_pointer: true,734+ keyboard: false,735+ text: true,736+ gamepad: false,737+ input_ack: false,738+ control: true,739+ }740+ );741+ assert_eq!(742+ InputAvailability::from_capabilities(&[Capability::Keyboard]),743+ InputAvailability {744+ relative_pointer: false,745+ pointer_position_feedback: false,746+ absolute_pointer: false,747+ keyboard: true,748+ text: false,749+ gamepad: false,750+ input_ack: false,751+ control: true,752+ }753+ );754+ assert_eq!(755+ InputAvailability::from_capabilities(&[Capability::Gamepad]),756+ InputAvailability {757+ relative_pointer: false,758+ pointer_position_feedback: false,759+ absolute_pointer: false,760+ keyboard: false,761+ text: false,762+ gamepad: true,763+ input_ack: false,764+ control: true,765+ }766+ );767+ assert!(768+ InputAvailability::from_capabilities(&[Capability::Gamepad, Capability::InputAck])769+ .input_ack770+ );771+ assert_eq!(772+ InputAvailability::from_capabilities(&[773+ Capability::RelativePointer,774+ Capability::PointerPositionFeedback,775+ ]),776+ InputAvailability {777+ relative_pointer: true,778+ pointer_position_feedback: true,779+ absolute_pointer: false,780+ keyboard: false,781+ text: false,782+ gamepad: false,783+ input_ack: false,784+ control: true,785+ }786+ );787+ }788+789+ 790+ fn pointer_feedback_without_input_ack_does_not_fail_ack_validation() {791+ let diagnostics = StdMutex::new(NetworkSessionDiagnostics::new(Instant::now(), false));792+ assert!(observe_input_ack_if_negotiated(&diagnostics, false, 0, Instant::now()).is_ok());793+ }794+795+ 796+ fn input_capability_fallbacks_remove_only_one_protocol_extension_at_a_time() {797+ let offers = input_capability_offers(InvitationGrants::ALL);798+ assert_eq!(offers.len(), 8);799+ let visibility = &offers[0];800+ let position = &offers[1];801+ let relative = &offers[2];802+ let inherited = &offers[3];803+804+ assert!(805+ offers[..4]806+ .iter()807+ .all(|offer| offer.contains(&Capability::InputAck))808+ );809+ assert!(810+ offers[4..]811+ .iter()812+ .all(|offer| !offer.contains(&Capability::InputAck))813+ );814+ assert!(visibility.contains(&Capability::PointerVisibilityFeedback));815+ assert!(visibility.contains(&Capability::PointerPositionFeedback));816+ assert!(!position.contains(&Capability::PointerVisibilityFeedback));817+ assert!(position.contains(&Capability::PointerPositionFeedback));818+ assert!(!relative.contains(&Capability::PointerVisibilityFeedback));819+ assert!(!relative.contains(&Capability::PointerPositionFeedback));820+ assert!(relative.contains(&Capability::RelativePointer));821+ assert!(!inherited.contains(&Capability::RelativePointer));822+ assert_eq!(823+ inherited.as_slice(),824+ &[825+ Capability::AbsolutePointer,826+ Capability::Keyboard,827+ Capability::Text,828+ Capability::Gamepad,829+ Capability::InputAck,830+ ]831+ );832+ }833+834+ 835+ fn local_invitation_grants_bound_input_offers_before_the_host_intersection() {836+ let view_only = input_capability_offers(InvitationGrants::VIEW);837+ assert!(view_only.iter().all(Vec::is_empty));838+839+ let pointer = input_capability_offers(840+ InvitationGrants::VIEW.union(InvitationGrants::POINTER_KEYBOARD),841+ );842+ assert!(pointer[0].contains(&Capability::Keyboard));843+ assert!(pointer[0].contains(&Capability::InputAck));844+ assert!(!pointer[0].contains(&Capability::Gamepad));845+846+ let gamepad =847+ input_capability_offers(InvitationGrants::VIEW.union(InvitationGrants::GAMEPAD));848+ assert_eq!(gamepad[0], vec![Capability::Gamepad, Capability::InputAck]);849+ assert_eq!(gamepad[1], vec![Capability::Gamepad]);850+ }851+852+ 853+ fn relative_pointer_accumulator_coalesces_chunks_and_resets() {854+ let mut accumulator = RelativePointerAccumulator::default();855+ accumulator.push(10, -20);856+ accumulator.push(5, 8);857+ assert_eq!(858+ accumulator.take(),859+ Some(InputEvent::MouseMoveRelative { dx: 15, dy: -12 })860+ );861+ assert_eq!(accumulator.take(), None);862+863+ accumulator.push(RELATIVE_POINTER_DELTA_MAX, RELATIVE_POINTER_DELTA_MIN);864+ accumulator.push(1_000, -1_000);865+ assert_eq!(866+ accumulator.take(),867+ Some(InputEvent::MouseMoveRelative {868+ dx: RELATIVE_POINTER_DELTA_MAX,869+ dy: RELATIVE_POINTER_DELTA_MIN,870+ })871+ );872+ assert_eq!(873+ accumulator.take(),874+ Some(InputEvent::MouseMoveRelative {875+ dx: 1_000,876+ dy: -1_000,877+ })878+ );879+ assert_eq!(accumulator.take(), None);880+ }881+882+ 883+ fn relative_motion_chunks_are_staged_immediately_before_a_following_button() {884+ let mut accumulator = RelativePointerAccumulator::default();885+ assert_eq!(886+ stage_relative_input(887+ &mut accumulator,888+ InputEvent::MouseMoveRelative {889+ dx: RELATIVE_POINTER_DELTA_MAX,890+ dy: RELATIVE_POINTER_DELTA_MIN,891+ }892+ ),893+ None894+ );895+ assert_eq!(896+ stage_relative_input(897+ &mut accumulator,898+ InputEvent::MouseMoveRelative { dx: 7, dy: -1 }899+ ),900+ None901+ );902+ assert_eq!(903+ stage_relative_input(&mut accumulator, InputEvent::MouseDown { b: 1 }),904+ Some(InputEvent::MouseDown { b: 1 })905+ );906+ assert_eq!(907+ accumulator.take(),908+ Some(InputEvent::MouseMoveRelative {909+ dx: RELATIVE_POINTER_DELTA_MAX,910+ dy: RELATIVE_POINTER_DELTA_MIN,911+ })912+ );913+ assert_eq!(914+ accumulator.take(),915+ Some(InputEvent::MouseMoveRelative { dx: 7, dy: -1 })916+ );917+ assert_eq!(accumulator.take(), None);918+ }919+}Not shown: this page already shows 6000 lines of diff.
Not shown: this page already shows 6000 lines of diff.
Not shown: this page already shows 6000 lines of diff.
Not shown: this page already shows 6000 lines of diff.
Not shown: this page already shows 6000 lines of diff.
Not shown: this page already shows 6000 lines of diff.
Not shown: this page already shows 6000 lines of diff.
Not shown: this page already shows 6000 lines of diff.
Not shown: this page already shows 6000 lines of diff.