* feat: prepare portal for format transitions * feat: transact encoder resolution changes Add revisioned resolution control with exact configured-IDR acknowledgement, per-frame dimensions, and discontinuity barriers. Extend the ignored Linux x264 smoke across downscale and restore transitions. * feat: adapt stream resolution to motion pressure * docs: define motion resolution acceptance * fix: compile Linux encoder controls
- commit
- 203e13e7c5c2a20a797aba4ad51cc0c9ffb2ebfd
- parent
- 42b178fc2f8ae702263ff5d022ff8bdcb26dce47
- author
- FelineStateMachine <dami.etoile@gmail.com>
| M | README.md | 17 | +- |
| M | crates/ | 812 | ++++++++++++++++++++++++++++-- |
| M | crates/ | 510 | ++++++++++++++++++- |
| M | docs/ | 32 | + |
| M | portal/ | 7 | +- |
| M | portal/ | 5 | +- |
| M | portal/ | 27 | +- |
| M | portal/ | 16 | +- |
| M | portal/ | 169 | +++++-- |
| A | portal/ | 68 | +++ |
| A | portal/ | 57 | ++ |
| M | portal/ | 12 | + |
| M | portal/ | 7 | + |
| M | scripts/ | 2 | +- |
| M | src-tauri/ | 42 | +- |
@@ -42,13 +42,18 @@ Steam and games.
4242 - Gamescope video keeps the proven external `gst-launch` pipeline as its4343 configuration default. Linux builds made explicitly with4444 `in-process-gstreamer` can opt into an in-process pipeline for bounded45- bitrate and force-keyframe control; ordinary release packages do not enable46- that feature yet.45+ bitrate, force-keyframe, and two-tier resolution control; ordinary release46+ packages do not enable that feature yet.4747 - Portal reports bounded receiver queue, drop, latency, and recovery telemetry4848 over a session-authenticated feedback stream. The host combines it with4949 trusted path and scheduler pressure, applies hysteresis, and commits an5050 in-process CBR bitrate decision only after exact encoder readback. External5151 encoder sessions remain explicitly advisory-only.52+- On the opt-in in-process backend, fresh damage-driven frame progress and53+ authenticated receiver pressure select either native 1280x800 or the exact54+ same-aspect 960x600 motion tier. A switch commits only when GStreamer emits a55+ target-size IDR with SPS/PPS; Portal then starts a new decoder epoch while56+ pointer mapping remains tied to the native Gamescope surface.5257 - Portal delivers encoded frames to WebCodecs through a raw Tauri binary5358 channel. The handoff is capped at four frames, the decode queue and5459 presentation queue at two, and a bounded watchdog recovers a suspended@@ -113,6 +118,14 @@ not authentication.
113118 and `applied=true` requires exact in-process encoder readback. Native Ubuntu114119 x264 control is CI-proven; Bazzite GstVA loss/relay acceptance and product115120 packaging remain in issue #8.121+10. **Motion-sensitive resolution implemented; hardware proof pending:** the122+ in-process backend lowers resolution during sustained damage-driven motion123+ or receiver pressure and restores native detail only after stillness,124+ clean feedback, and cooldown. Every transition is an exact configured-IDR125+ discontinuity with truthful per-frame dimensions. Portal reconfigures126+ WebCodecs atomically and preserves native pointer mapping. Ubuntu x264 CI127+ covers downscale and restore; repeated GstVA/Portal acceptance on Bazzite128+ remains in issue #9.116129 117130 ## Development118131 @@ -77,6 +77,11 @@ const ADAPTIVE_BITRATE_QUANTUM_KBPS: u32 = 250;
7777 const ADAPTIVE_BITRATE_CLEAN_WINDOWS: u8 = 10;7878 const ADAPTIVE_BITRATE_MODERATE_WINDOWS: u8 = 2;7979 const ADAPTIVE_BITRATE_COOLDOWN_WINDOWS: u8 = 10;80+const MOTION_RESOLUTION_MODERATE_WINDOWS: u8 = 2;81+const MOTION_RESOLUTION_CLEAN_WINDOWS: u8 = 3;82+const MOTION_RESOLUTION_ACTIVITY_FPS: u64 = 10;83+const MOTION_RESOLUTION_STILL_SETTLE: Duration = Duration::from_secs(2);84+const MOTION_RESOLUTION_UPSCALE_COOLDOWN: Duration = Duration::from_secs(3);8085 const FEEDBACK_FRESH_INTERVAL_MIN_MS: u16 = 750;8186 const FEEDBACK_FRESH_INTERVAL_MAX_MS: u16 = 1_500;8287 const FEEDBACK_FRESH_ARRIVAL_GAP_MIN: Duration = Duration::from_millis(750);@@ -98,6 +103,290 @@ enum FeedbackSeverity {
98103 Stale,99104 }100105 106+107+pub(crate) struct VideoDimensions {108+ pub width: u16,109+ pub height: u16,110+}111+112+impl VideoDimensions {113+ fn three_quarter(self) -> Result<Self> {114+ ensure!(115+ self.width.is_multiple_of(4) && self.height.is_multiple_of(4),116+ "native dimensions must be divisible by four for an exact three-quarter tier"117+ );118+ let reduced = Self {119+ width: self.width / 4 * 3,120+ height: self.height / 4 * 3,121+ };122+ ensure!(123+ reduced.width >= 64 && reduced.height >= 64,124+ "three-quarter dimensions are below the H.264 minimum"125+ );126+ ensure!(127+ reduced.width.is_multiple_of(2) && reduced.height.is_multiple_of(2),128+ "three-quarter H.264 dimensions must be even"129+ );130+ ensure!(131+ u32::from(self.width) * u32::from(reduced.height)132+ == u32::from(self.height) * u32::from(reduced.width),133+ "three-quarter dimensions must preserve the native aspect ratio"134+ );135+ Ok(reduced)136+ }137+}138+139+140+enum ResolutionTier {141+ Native,142+ Reduced,143+}144+145+146+enum ResolutionTrigger {147+ Hold,148+ Motion,149+ Pressure,150+ Recovery,151+}152+153+154+struct MotionResolutionDecision {155+ tier: ResolutionTier,156+ target: VideoDimensions,157+ trigger: ResolutionTrigger,158+ changed: bool,159+}160+161+/// Pure two-tier resolution policy. Fresh sequence progress from Gamescope's162+/// damage-driven stream is the bounded motion proxy while the authenticated163+/// feedback path supplies pressure. A downshift is safety-biased and164+/// immediate; an upscale requires both settled motion and fresh clean165+/// feedback, so stale reports can never restore detail.166+167+pub(crate) struct MotionResolutionPolicy {168+ native: VideoDimensions,169+ reduced: VideoDimensions,170+ tier: ResolutionTier,171+ motion_active: bool,172+ still_since: Option<Instant>,173+ pressure_active: bool,174+ moderate_windows: u8,175+ clean_windows: u8,176+ last_transition_at: Option<Instant>,177+ last_motion_sequence: Option<u64>,178+}179+180+impl MotionResolutionPolicy {181+ pub(crate) fn new(native: VideoDimensions) -> Result<Self> {182+ ensure!(183+ native.width >= 64 && native.height >= 64,184+ "native dimensions are too small"185+ );186+ ensure!(187+ native.width.is_multiple_of(2) && native.height.is_multiple_of(2),188+ "native H.264 dimensions must be even"189+ );190+ Ok(Self {191+ native,192+ reduced: native.three_quarter()?,193+ tier: ResolutionTier::Native,194+ motion_active: false,195+ still_since: None,196+ pressure_active: false,197+ moderate_windows: 0,198+ clean_windows: 0,199+ last_transition_at: None,200+ last_motion_sequence: None,201+ })202+ }203+204+ pub(crate) fn target(&self) -> VideoDimensions {205+ match self.tier {206+ ResolutionTier::Native => self.native,207+ ResolutionTier::Reduced => self.reduced,208+ }209+ }210+211+ /// Host motion hook. Repeated still observations preserve the original212+ /// settle boundary instead of indefinitely postponing recovery.213+ 214+ fn observe_motion(215+ &mut self,216+ motion_active: bool,217+ observed_at: Instant,218+ ) -> MotionResolutionDecision {219+ self.update_motion(motion_active, observed_at);220+ self.evaluate_target(observed_at)221+ }222+223+ fn update_motion(&mut self, motion_active: bool, observed_at: Instant) {224+ if motion_active {225+ self.motion_active = true;226+ self.still_since = None;227+ return;228+ }229+ if self.motion_active || self.still_since.is_none() {230+ self.still_since = Some(observed_at);231+ }232+ self.motion_active = false;233+ }234+235+ /// Observe one classified feedback window. Only fresh moderate/severe236+ /// pressure can initiate a downshift, and only fresh clean windows count237+ /// toward restoring native detail.238+ 239+ fn observe_feedback(240+ &mut self,241+ severity: FeedbackSeverity,242+ fresh: bool,243+ observed_at: Instant,244+ ) -> MotionResolutionDecision {245+ self.update_feedback(severity, fresh);246+ self.evaluate_target(observed_at)247+ }248+249+ /// Gamescope output is damage-driven, so fresh media sequence progress is250+ /// a conservative constant-space motion proxy for this first slice. A251+ /// future raw-frame observer can call `observe_motion` without changing252+ /// pressure hysteresis or encoder actuation.253+ fn observe_window(254+ &mut self,255+ report: &MediaFeedbackReportV1,256+ evaluation: &AdaptiveBitrateEvaluation,257+ observed_at: Instant,258+ ) -> MotionResolutionDecision {259+ let previous_sequence = self.last_motion_sequence;260+ if let Some(sequence) = report.last_sequence {261+ self.last_motion_sequence = Some(262+ self.last_motion_sequence263+ .map_or(sequence, |last| last.max(sequence)),264+ );265+ }266+ if evaluation.fresh {267+ let motion_active = report.last_sequence.is_some_and(|sequence| {268+ let delta = previous_sequence.map_or(0, |last| sequence.saturating_sub(last));269+ delta.saturating_mul(1_000)270+ >= MOTION_RESOLUTION_ACTIVITY_FPS.saturating_mul(u64::from(report.interval_ms))271+ });272+ self.update_motion(motion_active, observed_at);273+ }274+ self.update_feedback(evaluation.pressure_severity, evaluation.fresh);275+ self.evaluate_target(observed_at)276+ }277+278+ fn update_feedback(&mut self, severity: FeedbackSeverity, fresh: bool) {279+ if !fresh280+ || matches!(281+ severity,282+ FeedbackSeverity::InsufficientEvidence | FeedbackSeverity::Stale283+ )284+ {285+ self.moderate_windows = 0;286+ self.clean_windows = 0;287+ return;288+ }289+290+ match severity {291+ FeedbackSeverity::Severe => {292+ self.pressure_active = true;293+ self.moderate_windows = 0;294+ self.clean_windows = 0;295+ }296+ FeedbackSeverity::Moderate => {297+ self.clean_windows = 0;298+ self.moderate_windows = self.moderate_windows.saturating_add(1);299+ if self.moderate_windows >= MOTION_RESOLUTION_MODERATE_WINDOWS {300+ self.moderate_windows = 0;301+ self.pressure_active = true;302+ }303+ }304+ FeedbackSeverity::Clean => {305+ self.moderate_windows = 0;306+ self.clean_windows = self.clean_windows.saturating_add(1);307+ if self.clean_windows >= MOTION_RESOLUTION_CLEAN_WINDOWS {308+ self.pressure_active = false;309+ }310+ }311+ FeedbackSeverity::InsufficientEvidence | FeedbackSeverity::Stale => {}312+ }313+ }314+315+ fn evaluate_target(&mut self, observed_at: Instant) -> MotionResolutionDecision {316+ if self.motion_active {317+ self.downshift(observed_at, ResolutionTrigger::Motion)318+ } else if self.pressure_active {319+ self.downshift(observed_at, ResolutionTrigger::Pressure)320+ } else {321+ self.maybe_restore(observed_at)322+ }323+ }324+325+ fn commit_observation_from(&mut self, candidate: &Self) {326+ self.motion_active = candidate.motion_active;327+ self.still_since = candidate.still_since;328+ self.pressure_active = candidate.pressure_active;329+ self.moderate_windows = candidate.moderate_windows;330+ self.clean_windows = candidate.clean_windows;331+ self.last_motion_sequence = candidate.last_motion_sequence;332+ }333+334+ fn downshift(335+ &mut self,336+ observed_at: Instant,337+ trigger: ResolutionTrigger,338+ ) -> MotionResolutionDecision {339+ let changed = self.tier != ResolutionTier::Reduced;340+ if changed {341+ self.tier = ResolutionTier::Reduced;342+ self.last_transition_at = Some(observed_at);343+ }344+ MotionResolutionDecision {345+ tier: self.tier,346+ target: self.target(),347+ trigger,348+ changed,349+ }350+ }351+352+ fn maybe_restore(&mut self, observed_at: Instant) -> MotionResolutionDecision {353+ let still_settled = !self.motion_active354+ && self.still_since.is_some_and(|still_since| {355+ observed_at.saturating_duration_since(still_since) >= MOTION_RESOLUTION_STILL_SETTLE356+ });357+ let cooldown_complete = self.last_transition_at.is_none_or(|last| {358+ observed_at.saturating_duration_since(last) >= MOTION_RESOLUTION_UPSCALE_COOLDOWN359+ });360+ let clean = self.clean_windows >= MOTION_RESOLUTION_CLEAN_WINDOWS;361+ if self.tier == ResolutionTier::Reduced362+ && still_settled363+ && cooldown_complete364+ && clean365+ && !self.pressure_active366+ {367+ self.tier = ResolutionTier::Native;368+ self.clean_windows = 0;369+ self.last_transition_at = Some(observed_at);370+ return MotionResolutionDecision {371+ tier: self.tier,372+ target: self.target(),373+ trigger: ResolutionTrigger::Recovery,374+ changed: true,375+ };376+ }377+ self.hold()378+ }379+380+ fn hold(&self) -> MotionResolutionDecision {381+ MotionResolutionDecision {382+ tier: self.tier,383+ target: self.target(),384+ trigger: ResolutionTrigger::Hold,385+ changed: false,386+ }387+ }388+}389+101390 102391 struct FeedbackIngressCursor {103392 last_report_id: Option<u64>,@@ -211,6 +500,13 @@ struct ShadowBitrateController {
211500 cooldown_windows: u8,212501 }213502 503+504+struct AdaptiveBitrateEvaluation {505+ decision: AdaptiveBitrateDecisionV1,506+ pressure_severity: FeedbackSeverity,507+ fresh: bool,508+}509+214510 impl ShadowBitrateController {215511 fn new(configured_ceiling_kbps: u32, telemetry: MediaV3TelemetrySnapshot) -> Self {216512 let ceiling_kbps = quantize_down(configured_ceiling_kbps).max(ADAPTIVE_BITRATE_FLOOR_KBPS);@@ -230,18 +526,18 @@ impl ShadowBitrateController {
230526 }231527 }232528 233- fn decide(529+ fn evaluate(234530 &mut self,235531 report: &MediaFeedbackReportV1,236532 telemetry: MediaV3TelemetrySnapshot,237533 received_at: Instant,238- ) -> Result<AdaptiveBitrateDecisionV1> {534+ ) -> Result<AdaptiveBitrateEvaluation> {239535 ensure!(240536 self.last_report_id241537 .is_none_or(|last| report.report_id > last),242538 "feedback report IDs must increase monotonically"243539 );244- let (severity, reasons, stale, trusted_host_pressure) =540+ let (severity, pressure_severity, reasons, stale, trusted_host_pressure) =245541 self.classify(report, telemetry, received_at);246542 let mut state = AdaptiveBitrateStateV1::Hold;247543 @@ -331,7 +627,22 @@ impl ShadowBitrateController {
331627 .map_or(sequence, |last| last.max(sequence)),332628 );333629 }334- Ok(decision)630+ Ok(AdaptiveBitrateEvaluation {631+ decision,632+ pressure_severity,633+ fresh: !stale,634+ })635+ }636+637+ 638+ fn decide(639+ &mut self,640+ report: &MediaFeedbackReportV1,641+ telemetry: MediaV3TelemetrySnapshot,642+ received_at: Instant,643+ ) -> Result<AdaptiveBitrateDecisionV1> {644+ self.evaluate(report, telemetry, received_at)645+ .map(|evaluation| evaluation.decision)335646 }336647 337648 fn classify(@@ -339,7 +650,13 @@ impl ShadowBitrateController {
339650 report: &MediaFeedbackReportV1,340651 telemetry: MediaV3TelemetrySnapshot,341652 received_at: Instant,342- ) -> (FeedbackSeverity, AdaptiveBitrateReasonFlagsV1, bool, bool) {653+ ) -> (654+ FeedbackSeverity,655+ FeedbackSeverity,656+ AdaptiveBitrateReasonFlagsV1,657+ bool,658+ bool,659+ ) {343660 let stale = !(FEEDBACK_FRESH_INTERVAL_MIN_MS..=FEEDBACK_FRESH_INTERVAL_MAX_MS)344661 .contains(&report.interval_ms)345662 || self.last_report_received_at.is_none()@@ -422,7 +739,7 @@ impl ShadowBitrateController {
422739 if !trusted_host_pressure {423740 severity = FeedbackSeverity::Stale;424741 }425- return (severity, reasons, true, trusted_host_pressure);742+ return (severity, severity, reasons, true, trusted_host_pressure);426743 }427744 428745 let receiver_drops = report@@ -517,6 +834,7 @@ impl ShadowBitrateController {
517834 );518835 }519836 837+ let pressure_severity = severity;520838 if severity == FeedbackSeverity::Clean {521839 let sequence_progressed = self522840 .last_sequence@@ -533,7 +851,13 @@ impl ShadowBitrateController {
533851 severity = FeedbackSeverity::InsufficientEvidence;534852 }535853 }536- (severity, reasons, false, trusted_host_pressure)854+ (855+ severity,856+ pressure_severity,857+ reasons,858+ false,859+ trusted_host_pressure,860+ )537861 }538862 539863 fn commit_observation_from(&mut self, candidate: &Self) {@@ -655,6 +979,13 @@ struct AdaptiveEncoderProposal {
655979 force_keyframe_revision: Option<u64>,656980 }657981 982+983+struct ResolutionEncoderProposal {984+ control: EncoderControl,985+ target: VideoDimensions,986+ revision: u64,987+}988+658989 impl MediaV3Telemetry {659990 fn snapshot(&self) -> MediaV3TelemetrySnapshot {660991 MediaV3TelemetrySnapshot {@@ -804,6 +1135,30 @@ impl SessionRegistry {
8041135 }))8051136 }8061137 1138+ fn propose_resolution_update(1139+ &self,1140+ remote: EndpointId,1141+ session_id: u64,1142+ target: VideoDimensions,1143+ ) -> Result<Option<ResolutionEncoderProposal>> {1144+ let active = self.active.lock().expect("session registry poisoned");1145+ let session = active1146+ .as_ref()1147+ .filter(|session| {1148+ session.media_active && session.remote == remote && session.session_id == session_id1149+ })1150+ .context("resolution update does not match the active media session")?;1151+ let Some(control) = session.encoder_control.clone() else {1152+ return Ok(None);1153+ };1154+ let revision = control.request_resolution(target.width, target.height)?;1155+ Ok(Some(ResolutionEncoderProposal {1156+ control,1157+ target,1158+ revision,1159+ }))1160+ }1161+8071162 fn claim_feedback(8081163 self: &Arc<Self>,8091164 remote: EndpointId,@@ -2623,6 +2978,14 @@ async fn serve_media_feedback(
26232978 let mut reader_finished = false;26242979 let mut receiver_open = true;26252980 let mut controller = ShadowBitrateController::new(ceiling_kbps, lease.telemetry.snapshot());2981+ let mut resolution_controller = if encoder_actuation_available {2982+ Some(MotionResolutionPolicy::new(VideoDimensions {2983+ width: u16::try_from(config.width).context("native width exceeds protocol")?,2984+ height: u16::try_from(config.height).context("native height exceeds protocol")?,2985+ })?)2986+ } else {2987+ None2988+ };26262989 let mut feedback_cursor = FeedbackIngressCursor::default();26272990 26282991 let result: Result<()> = loop {@@ -2665,15 +3028,24 @@ async fn serve_media_feedback(
26653028 // arriving during an encoder wait accumulate beyond it.26663029 feedback_cursor = next_cursor;26673030 let telemetry = lease.telemetry.snapshot();3031+ let observed_at = Instant::now();26683032 let mut candidate = controller.clone();2669- let mut decision = candidate.decide(&report, telemetry, Instant::now())?;3033+ let evaluation = candidate.evaluate(&report, telemetry, observed_at)?;3034+ let mut decision = evaluation.decision;3035+ let mut resolution_candidate = resolution_controller.clone();3036+ let resolution_decision = resolution_candidate3037+ .as_mut()3038+ .map(|policy| policy.observe_window(&report, &evaluation, observed_at));26703039 let changes_bitrate = decision.state != AdaptiveBitrateStateV1::Hold;2671- let can_actuate = changes_bitrate && encoder_actuation_available;3040+ let changes_resolution = resolution_decision.is_some_and(|change| change.changed);3041+ let can_actuate_bitrate = changes_bitrate && encoder_actuation_available;3042+ let can_actuate_resolution = changes_resolution && encoder_actuation_available;26723043 let mut recovery_acknowledged = None;2673- if can_actuate {3044+ let bitrate_proposal = if can_actuate_bitrate {26743045 let force_keyframe =2675- adaptive_recovery_keyframe_required(&report, &decision);2676- let proposal = match sessions.propose_adaptive_encoder_update(3046+ adaptive_recovery_keyframe_required(&report, &decision)3047+ && !changes_resolution;3048+ match sessions.propose_adaptive_encoder_update(26773049 remote,26783050 lease.session_id,26793051 decision.target_kbps,@@ -2697,8 +3069,39 @@ async fn serve_media_feedback(
26973069 );26983070 None26993071 }2700- };2701- if let Some(proposal) = proposal {3072+ }3073+ } else {3074+ None3075+ };3076+ let resolution_proposal = if can_actuate_resolution {3077+ let target = resolution_decision3078+ .expect("resolution change is required for actuation")3079+ .target;3080+ match sessions.propose_resolution_update(remote, lease.session_id, target) {3081+ Ok(Some(proposal)) => Some(proposal),3082+ Ok(None) => {3083+ warn!(3084+ %remote,3085+ session_id = lease.session_id,3086+ "active adaptive session lost encoder control; retaining committed resolution"3087+ );3088+ None3089+ }3090+ Err(error) => {3091+ warn!(3092+ %remote,3093+ session_id = lease.session_id,3094+ %error,3095+ "adaptive resolution proposal failed; retaining committed resolution"3096+ );3097+ None3098+ }3099+ }3100+ } else {3101+ None3102+ };3103+3104+ if let Some(proposal) = bitrate_proposal {27023105 match commit_adaptive_encoder_proposal(27033106 sessions,27043107 remote,@@ -2743,7 +3146,8 @@ async fn serve_media_feedback(
27433146 }27443147 }27453148 }2746- }3149+ }3150+ if can_actuate_bitrate {27473151 if !decision.applied {27483152 controller.commit_observation_from(&candidate);27493153 }@@ -2753,6 +3157,44 @@ async fn serve_media_feedback(
27533157 // need encoder actuation.27543158 controller = candidate;27553159 }3160+3161+ let mut resolution_applied = None;3162+ if let Some(proposal) = resolution_proposal {3163+ match commit_resolution_encoder_proposal(3164+ sessions,3165+ remote,3166+ lease.session_id,3167+ &proposal,3168+ )3169+ .await3170+ {3171+ AdaptiveEncoderCommit::GenerationEnded => break Ok(()),3172+ AdaptiveEncoderCommit::NotApplied(error) => {3173+ resolution_applied = Some(false);3174+ warn!(3175+ %remote,3176+ session_id = lease.session_id,3177+ target_width = proposal.target.width,3178+ target_height = proposal.target.height,3179+ %error,3180+ "adaptive resolution application failed; retaining committed resolution"3181+ );3182+ }3183+ AdaptiveEncoderCommit::Applied => {3184+ resolution_applied = Some(true);3185+ resolution_controller = resolution_candidate.clone();3186+ }3187+ }3188+ }3189+ if can_actuate_resolution && resolution_applied != Some(true) {3190+ if let (Some(committed), Some(candidate)) =3191+ (resolution_controller.as_mut(), resolution_candidate.as_ref())3192+ {3193+ committed.commit_observation_from(candidate);3194+ }3195+ } else if !changes_resolution {3196+ resolution_controller = resolution_candidate;3197+ }27563198 tokio::time::timeout(27573199 FEEDBACK_WRITE_TIMEOUT,27583200 write_adaptive_bitrate_decision_v1(&mut send, &decision),@@ -2769,12 +3211,16 @@ async fn serve_media_feedback(
27693211 reasons = decision.reasons.bits(),27703212 applied = decision.applied,27713213 ?recovery_acknowledged,3214+ resolution_width = resolution_decision.map(|change| change.target.width),3215+ resolution_height = resolution_decision.map(|change| change.target.height),3216+ resolution_trigger = ?resolution_decision.map(|change| change.trigger),3217+ ?resolution_applied,27723218 path_rtt_micros = telemetry.selected_path_rtt_micros,27733219 path_lost_packets = telemetry.selected_path_lost_packets,27743220 path_congestion_events = telemetry.selected_path_congestion_events,27753221 scheduler_cancellations = telemetry.scheduler_cancellations,27763222 send_failures = telemetry.send_failures,2777- mode = if can_actuate { "active" } else { "shadow" },3223+ mode = if encoder_actuation_available { "active" } else { "shadow" },27783224 "adaptive bitrate decision"27793225 );27803226 }@@ -2931,6 +3377,48 @@ async fn commit_adaptive_encoder_proposal(
29313377 AdaptiveEncoderCommit::Applied29323378 }29333379 3380+async fn commit_resolution_encoder_proposal(3381+ sessions: &SessionRegistry,3382+ remote: EndpointId,3383+ session_id: u64,3384+ proposal: &ResolutionEncoderProposal,3385+) -> AdaptiveEncoderCommit {3386+ let result = await_while_session_active(3387+ sessions,3388+ remote,3389+ session_id,3390+ proposal.control.wait_for_resolution_applied(3391+ proposal.revision,3392+ proposal.target.width,3393+ proposal.target.height,3394+ ENCODER_CONTROL_COMMIT_TIMEOUT,3395+ ),3396+ )3397+ .await;3398+ let Some(result) = result else {3399+ return AdaptiveEncoderCommit::GenerationEnded;3400+ };3401+ match result {3402+ Ok(status)3403+ if status.applied_resolution_revision == Some(proposal.revision)3404+ && status.applied_width == Some(proposal.target.width)3405+ && status.applied_height == Some(proposal.target.height) =>3406+ {3407+ AdaptiveEncoderCommit::Applied3408+ }3409+ Ok(status) => AdaptiveEncoderCommit::NotApplied(anyhow::anyhow!(3410+ "encoder resolution commit mismatch: revision {:?}, dimensions {:?}x{:?}, expected revision {} at {}x{}",3411+ status.applied_resolution_revision,3412+ status.applied_width,3413+ status.applied_height,3414+ proposal.revision,3415+ proposal.target.width,3416+ proposal.target.height3417+ )),3418+ Err(error) => AdaptiveEncoderCommit::NotApplied(error),3419+ }3420+}3421+29343422 async fn await_adaptive_recovery_keyframe(29353423 sessions: &SessionRegistry,29363424 remote: EndpointId,@@ -3088,28 +3576,7 @@ async fn serve_media(
30883576 break;30893577 }30903578 };3091- let mut flags = FrameFlags::NONE;3092- if frame.keyframe {3093- flags = flags.union(FrameFlags::KEYFRAME);3094- }3095- if frame.codec_config {3096- flags = flags.union(FrameFlags::CODEC_CONFIG);3097- }3098- if discontinuity {3099- flags = flags.union(FrameFlags::DISCONTINUITY);3100- }3101- let width = u16::try_from(config.width).context("width exceeds protocol")?;3102- let height = u16::try_from(config.height).context("height exceeds protocol")?;3103- let header = MediaFrameHeader::h264(3104- width,3105- height,3106- frame.data.len(),3107- frame.sequence,3108- frame.capture_timestamp_micros,3109- frame.presentation_timestamp_micros,3110- flags,3111- )?;3112- let media_frame = MediaFrame::new(header, frame.data.as_ref().to_vec())?;3579+ let media_frame = media_frame_for_encoded(&config, &frame, discontinuity)?;31133580 tokio::time::timeout(31143581 MEDIA_WRITE_TIMEOUT,31153582 write_media_frame(&mut send, &media_frame),@@ -3476,7 +3943,7 @@ async fn run_media_v2_session(
34763943 }34773944 34783945 fn media_frame_for_encoded(3479- config: &HostConfig,3946+ _config: &HostConfig,34803947 frame: &EncodedFrame,34813948 discontinuity: bool,34823949 ) -> Result<MediaFrame> {@@ -3487,14 +3954,12 @@ fn media_frame_for_encoded(
34873954 if frame.codec_config {34883955 flags = flags.union(FrameFlags::CODEC_CONFIG);34893956 }3490- if discontinuity {3957+ if discontinuity || frame.discontinuity {34913958 flags = flags.union(FrameFlags::DISCONTINUITY);34923959 }3493- let width = u16::try_from(config.width).context("width exceeds protocol")?;3494- let height = u16::try_from(config.height).context("height exceeds protocol")?;34953960 let header = MediaFrameHeader::h264(3496- width,3497- height,3961+ frame.width,3962+ frame.height,34983963 frame.data.len(),34993964 frame.sequence,35003965 frame.capture_timestamp_micros,@@ -3874,7 +4339,7 @@ fn media_v3_delivery_timeout_ms(framerate: u32) -> u32 {
38744339 }38754340 38764341 fn media_v3_object_for_encoded(3877- config: &HostConfig,4342+ _config: &HostConfig,38784343 frame: &EncodedFrame,38794344 position: MediaV3ObjectPosition,38804345 discontinuity: bool,@@ -3887,7 +4352,7 @@ fn media_v3_object_for_encoded(
38874352 if frame.codec_config {38884353 flags = flags.union(FrameFlags::CODEC_CONFIG);38894354 }3890- if discontinuity {4355+ if discontinuity || frame.discontinuity {38914356 flags = flags.union(FrameFlags::DISCONTINUITY);38924357 }38934358 let publisher_priority = if position.object_id == 0 {@@ -3896,8 +4361,8 @@ fn media_v3_object_for_encoded(
38964361 MEDIA_V3_DELTA_PUBLISHER_PRIORITY38974362 };38984363 let header = MediaObjectHeaderV3::h264(3899- u16::try_from(config.width).context("width exceeds protocol")?,3900- u16::try_from(config.height).context("height exceeds protocol")?,4364+ frame.width,4365+ frame.height,39014366 frame.data.len(),39024367 publisher_priority,39034368 flags,@@ -4408,6 +4873,233 @@ mod tests {
44084873 .unwrap()44094874 }44104875 4876+ fn native_dimensions() -> VideoDimensions {4877+ VideoDimensions {4878+ width: 1_280,4879+ height: 800,4880+ }4881+ }4882+4883+ 4884+ fn motion_resolution_tier_is_exact_even_and_same_aspect() {4885+ let native = native_dimensions();4886+ let policy = MotionResolutionPolicy::new(native).unwrap();4887+ assert_eq!(policy.target(), native);4888+ assert_eq!(4889+ policy.reduced,4890+ VideoDimensions {4891+ width: 960,4892+ height: 600,4893+ }4894+ );4895+ assert_eq!(4896+ u32::from(native.width) * u32::from(policy.reduced.height),4897+ u32::from(native.height) * u32::from(policy.reduced.width)4898+ );4899+ assert!(policy.reduced.width.is_multiple_of(2));4900+ assert!(policy.reduced.height.is_multiple_of(2));4901+4902+ assert!(4903+ MotionResolutionPolicy::new(VideoDimensions {4904+ width: 1_278,4905+ height: 800,4906+ })4907+ .is_err()4908+ );4909+ assert!(4910+ MotionResolutionPolicy::new(VideoDimensions {4911+ width: 80,4912+ height: 64,4913+ })4914+ .is_err()4915+ );4916+ }4917+4918+ 4919+ fn motion_downshifts_immediately_and_recovers_only_after_settle_and_clean_windows() {4920+ let now = Instant::now();4921+ let mut policy = MotionResolutionPolicy::new(native_dimensions()).unwrap();4922+ let down = policy.observe_motion(true, now);4923+ assert_eq!(down.tier, ResolutionTier::Reduced);4924+ assert_eq!(down.trigger, ResolutionTrigger::Motion);4925+ assert!(down.changed);4926+4927+ let still = policy.observe_motion(false, now + Duration::from_secs(1));4928+ assert!(!still.changed);4929+ for second in [2, 3] {4930+ assert!(4931+ !policy4932+ .observe_feedback(4933+ FeedbackSeverity::Clean,4934+ true,4935+ now + Duration::from_secs(second),4936+ )4937+ .changed4938+ );4939+ }4940+ let recovered =4941+ policy.observe_feedback(FeedbackSeverity::Clean, true, now + Duration::from_secs(4));4942+ assert_eq!(recovered.tier, ResolutionTier::Native);4943+ assert_eq!(recovered.target, native_dimensions());4944+ assert_eq!(recovered.trigger, ResolutionTrigger::Recovery);4945+ assert!(recovered.changed);4946+ }4947+4948+ 4949+ fn repeated_still_observations_do_not_postpone_resolution_recovery() {4950+ let now = Instant::now();4951+ let mut policy = MotionResolutionPolicy::new(native_dimensions()).unwrap();4952+ policy.observe_motion(true, now);4953+ policy.observe_motion(false, now + Duration::from_secs(1));4954+ policy.observe_feedback(FeedbackSeverity::Clean, true, now + Duration::from_secs(2));4955+ policy.observe_motion(false, now + Duration::from_millis(2_500));4956+ policy.observe_feedback(FeedbackSeverity::Clean, true, now + Duration::from_secs(3));4957+ let recovered =4958+ policy.observe_feedback(FeedbackSeverity::Clean, true, now + Duration::from_secs(4));4959+ assert!(recovered.changed);4960+ assert_eq!(recovered.tier, ResolutionTier::Native);4961+ }4962+4963+ 4964+ fn resolution_pressure_requires_fresh_hysteresis_but_severe_is_immediate() {4965+ let now = Instant::now();4966+ let mut moderate = MotionResolutionPolicy::new(native_dimensions()).unwrap();4967+ moderate.observe_motion(false, now);4968+ assert!(4969+ !moderate4970+ .observe_feedback(4971+ FeedbackSeverity::Moderate,4972+ true,4973+ now + Duration::from_secs(1),4974+ )4975+ .changed4976+ );4977+ assert!(4978+ !moderate4979+ .observe_feedback(4980+ FeedbackSeverity::Moderate,4981+ false,4982+ now + Duration::from_secs(2),4983+ )4984+ .changed4985+ );4986+ assert!(4987+ !moderate4988+ .observe_feedback(4989+ FeedbackSeverity::Moderate,4990+ true,4991+ now + Duration::from_secs(3),4992+ )4993+ .changed4994+ );4995+ let down = moderate.observe_feedback(4996+ FeedbackSeverity::Moderate,4997+ true,4998+ now + Duration::from_secs(4),4999+ );5000+ assert!(down.changed);5001+ assert_eq!(down.trigger, ResolutionTrigger::Pressure);5002+5003+ let mut severe = MotionResolutionPolicy::new(native_dimensions()).unwrap();5004+ let immediate =5005+ severe.observe_feedback(FeedbackSeverity::Severe, true, now + Duration::from_secs(1));5006+ assert!(immediate.changed);5007+ assert_eq!(immediate.tier, ResolutionTier::Reduced);5008+ }5009+5010+ 5011+ fn stale_feedback_cannot_restore_native_resolution() {5012+ let now = Instant::now();5013+ let mut policy = MotionResolutionPolicy::new(native_dimensions()).unwrap();5014+ policy.observe_motion(false, now);5015+ policy.observe_feedback(FeedbackSeverity::Severe, true, now + Duration::from_secs(1));5016+ for second in [2, 3, 4, 5] {5017+ let decision = policy.observe_feedback(5018+ FeedbackSeverity::Clean,5019+ false,5020+ now + Duration::from_secs(second),5021+ );5022+ assert_eq!(decision.tier, ResolutionTier::Reduced);5023+ assert!(!decision.changed);5024+ }5025+ assert!(policy.pressure_active);5026+5027+ for second in [6, 7] {5028+ assert!(5029+ !policy5030+ .observe_feedback(5031+ FeedbackSeverity::Clean,5032+ true,5033+ now + Duration::from_secs(second),5034+ )5035+ .changed5036+ );5037+ }5038+ let recovered =5039+ policy.observe_feedback(FeedbackSeverity::Clean, true, now + Duration::from_secs(8));5040+ assert!(recovered.changed);5041+ assert_eq!(recovered.tier, ResolutionTier::Native);5042+ }5043+5044+ 5045+ fn motion_keeps_resolution_reduced_after_pressure_clears() {5046+ let now = Instant::now();5047+ let mut policy = MotionResolutionPolicy::new(native_dimensions()).unwrap();5048+ policy.observe_motion(true, now);5049+ policy.observe_feedback(FeedbackSeverity::Severe, true, now + Duration::from_secs(1));5050+ for second in 2..=5 {5051+ let decision = policy.observe_feedback(5052+ FeedbackSeverity::Clean,5053+ true,5054+ now + Duration::from_secs(second),5055+ );5056+ assert_eq!(decision.tier, ResolutionTier::Reduced);5057+ assert!(!decision.changed);5058+ }5059+ assert!(!policy.pressure_active);5060+ assert!(policy.motion_active);5061+ }5062+5063+ 5064+ fn damage_driven_motion_proxy_seeds_then_downshifts_and_stale_blocks_upscale() {5065+ let now = Instant::now();5066+ let telemetry = MediaV3TelemetrySnapshot::default();5067+ let mut bitrate = ShadowBitrateController::new(12_000, telemetry);5068+ let mut resolution = MotionResolutionPolicy::new(native_dimensions()).unwrap();5069+5070+ let mut first = clean_feedback(1);5071+ first.last_sequence = Some(100);5072+ let first_evaluation = bitrate.evaluate(&first, telemetry, now).unwrap();5073+ assert!(!first_evaluation.fresh);5074+ let seeded = resolution.observe_window(&first, &first_evaluation, now);5075+ assert_eq!(seeded.tier, ResolutionTier::Native);5076+ assert!(!seeded.changed);5077+5078+ let mut second = clean_feedback(2);5079+ second.last_sequence = Some(110);5080+ let second_evaluation = bitrate5081+ .evaluate(&second, telemetry, now + Duration::from_secs(1))5082+ .unwrap();5083+ assert!(second_evaluation.fresh);5084+ let down =5085+ resolution.observe_window(&second, &second_evaluation, now + Duration::from_secs(1));5086+ assert_eq!(down.trigger, ResolutionTrigger::Motion);5087+ assert_eq!(down.tier, ResolutionTier::Reduced);5088+ assert!(down.changed);5089+5090+ let mut stale = clean_feedback(3);5091+ stale.interval_ms = 5_000;5092+ stale.last_sequence = Some(110);5093+ let stale_evaluation = bitrate5094+ .evaluate(&stale, telemetry, now + Duration::from_secs(5))5095+ .unwrap();5096+ assert!(!stale_evaluation.fresh);5097+ let held =5098+ resolution.observe_window(&stale, &stale_evaluation, now + Duration::from_secs(5));5099+ assert_eq!(held.tier, ResolutionTier::Reduced);5100+ assert!(!held.changed);5101+ }5102+44115103 44125104 fn shadow_controller_immediately_decreases_for_severe_feedback_and_stays_bounded() {44135105 let mut controller =@@ -4828,11 +5520,14 @@ mod tests {
48285520 ) -> EncodedFrame {48295521 EncodedFrame {48305522 sequence,5523+ width: 1_280,5524+ height: 800,48315525 capture_timestamp_micros: sequence,48325526 presentation_timestamp_micros: sequence as i64,48335527 observed_at: Instant::now(),48345528 keyframe,48355529 codec_config,5530+ discontinuity: false,48365531 data: Arc::from(vec![sequence as u8; payload_len]),48375532 }48385533 }@@ -5444,11 +6139,14 @@ mod tests {
54446139 let now = Instant::now();54456140 let frame = EncodedFrame {54466141 sequence: 0,6142+ width: 1_280,6143+ height: 800,54476144 capture_timestamp_micros: 0,54486145 presentation_timestamp_micros: 0,54496146 observed_at: now,54506147 keyframe: true,54516148 codec_config: true,6149+ discontinuity: false,54526150 data: Arc::from([1_u8, 2, 3]),54536151 };54546152 let (_frame_sender, mut frame_receiver) = tokio::sync::watch::channel(Some(frame.clone()));@@ -6204,11 +6902,14 @@ mod tests {
62046902 fn current_gop_replay_is_complete_and_skips_already_sent_frames() {62056903 let frame = |sequence, keyframe| EncodedFrame {62066904 sequence,6905+ width: 1_280,6906+ height: 800,62076907 capture_timestamp_micros: sequence,62086908 presentation_timestamp_micros: sequence as i64,62096909 observed_at: std::time::Instant::now(),62106910 keyframe,62116911 codec_config: keyframe,6912+ discontinuity: false,62126913 data: Arc::from([sequence as u8]),62136914 };62146915 @@ -6250,11 +6951,14 @@ mod tests {
62506951 let old_observation = observed_now - Duration::from_secs(1);62516952 let frame = |sequence, keyframe| EncodedFrame {62526953 sequence,6954+ width: 1_280,6955+ height: 800,62536956 capture_timestamp_micros: sequence,62546957 presentation_timestamp_micros: sequence as i64,62556958 observed_at: old_observation,62566959 keyframe,62576960 codec_config: keyframe,6961+ discontinuity: false,62586962 data: Arc::from([sequence as u8]),62596963 };62606964 let mut cursor = MediaReplayCursor::default();@@ -6296,11 +7000,14 @@ mod tests {
62967000 let old_observation = replay_started_at - Duration::from_secs(1);62977001 let frame = |sequence, keyframe| EncodedFrame {62987002 sequence,7003+ width: 1_280,7004+ height: 800,62997005 capture_timestamp_micros: sequence,63007006 presentation_timestamp_micros: sequence as i64,63017007 observed_at: old_observation,63027008 keyframe,63037009 codec_config: keyframe,7010+ discontinuity: false,63047011 data: Arc::from([sequence as u8]),63057012 };63067013 let mut cursor = MediaReplayCursor::default();@@ -6346,11 +7053,14 @@ mod tests {
63467053 assert_eq!(maximum_replay_age, Duration::from_nanos(33_333_334));63477054 let frame = EncodedFrame {63487055 sequence: 11,7056+ width: 1_280,7057+ height: 800,63497058 capture_timestamp_micros: 11,63507059 presentation_timestamp_micros: 11,63517060 observed_at: observed_now - maximum_replay_age,63527061 keyframe: false,63537062 codec_config: false,7063+ discontinuity: false,63547064 data: Arc::from([11]),63557065 };63567066 let mut cursor = MediaReplayCursor {@@ -6375,11 +7085,14 @@ mod tests {
63757085 let maximum_replay_age = maximum_media_replay_age(60);63767086 let frame = EncodedFrame {63777087 sequence: 11,7088+ width: 1_280,7089+ height: 800,63787090 capture_timestamp_micros: 11,63797091 presentation_timestamp_micros: 11,63807092 observed_at: observed_now - maximum_replay_age - Duration::from_nanos(1),63817093 keyframe: false,63827094 codec_config: false,7095+ discontinuity: false,63837096 data: Arc::from([11]),63847097 };63857098 let mut cursor = MediaReplayCursor {@@ -6405,11 +7118,14 @@ mod tests {
64057118 let maximum_replay_age = maximum_media_replay_age(60);64067119 let frame = |sequence, keyframe| EncodedFrame {64077120 sequence,7121+ width: 1_280,7122+ height: 800,64087123 capture_timestamp_micros: sequence,64097124 presentation_timestamp_micros: sequence as i64,64107125 observed_at: observed_now,64117126 keyframe,64127127 codec_config: keyframe,7128+ discontinuity: false,64137129 data: Arc::from([sequence as u8]),64147130 };64157131 let mut cursor = MediaReplayCursor {@@ -19,6 +19,9 @@ use crate::config::{
1919 GamescopeEncoderBackend, GamescopePipewireConfig, HostConfig, VaapiRateControl,2020 };2121 22+23+use gstreamer::prelude::ObjectExt;24+2225 2326 use std::os::unix::fs::{FileTypeExt, PermissionsExt};2427 @@ -29,6 +32,8 @@ const MAX_INSPECT_OUTPUT: usize = 1024 * 1024;
2932 const MAX_DIAGNOSTIC_OUTPUT: usize = 512 * 1024;3033 const PREFLIGHT_TIMEOUT: Duration = Duration::from_secs(5);3134 const MAX_ENCODER_BITRATE_KBPS: u32 = 100_000;35+36+const RESOLUTION_APPLY_TIMEOUT: Duration = Duration::from_secs(2);3237 3338 3439 struct RevisedBitrate {@@ -36,10 +41,18 @@ struct RevisedBitrate {
3641 kbps: u32,3742 }3843 44+45+struct RevisedResolution {46+ revision: u64,47+ width: u16,48+ height: u16,49+}50+3951 4052 struct EncoderControlDesired {4153 revision: u64,4254 bitrate: Option<RevisedBitrate>,55+ resolution: Option<RevisedResolution>,4356 force_keyframe_revision: Option<u64>,4457 }4558 @@ -48,6 +61,10 @@ pub struct EncoderControlStatus {
4861 pub latest_revision: u64,4962 pub applied_bitrate_revision: Option<u64>,5063 pub applied_bitrate_kbps: Option<u32>,64+ pub requested_resolution_revision: Option<u64>,65+ pub applied_resolution_revision: Option<u64>,66+ pub applied_width: Option<u16>,67+ pub applied_height: Option<u16>,5168 pub requested_force_keyframe_revision: Option<u64>,5269 pub acknowledged_force_keyframe_revision: Option<u64>,5370 }@@ -118,6 +135,27 @@ impl EncoderControl {
118135 Ok(revision)119136 }120137 138+ pub fn request_resolution(&self, width: u16, height: u16) -> Result<u64> {139+ ensure!(140+ width >= 64 && height >= 64,141+ "encoder dimensions must be at least 64x64"142+ );143+ ensure!(144+ width.is_multiple_of(2) && height.is_multiple_of(2),145+ "H.264 encoder dimensions must be even"146+ );147+ let revision = self.next_revision()?;148+ self.desired.send_modify(|desired| {149+ desired.revision = revision;150+ desired.resolution = Some(RevisedResolution {151+ revision,152+ width,153+ height,154+ });155+ });156+ Ok(revision)157+ }158+121159 pub fn status(&self) -> EncoderControlStatus {122160 *self.status.borrow()123161 }@@ -186,6 +224,55 @@ impl EncoderControl {
186224 .await187225 .with_context(|| format!("timed out waiting for force-keyframe revision {revision}"))?188226 }227+228+ /// Wait until the exact target dimensions emerge from the encoder on an229+ /// independently decodable access unit. A property write or caps event is230+ /// not an application acknowledgement.231+ pub async fn wait_for_resolution_applied(232+ &self,233+ revision: u64,234+ width: u16,235+ height: u16,236+ timeout: Duration,237+ ) -> Result<EncoderControlStatus> {238+ let mut status = self.status.clone();239+ let wait = async {240+ loop {241+ let snapshot = *status.borrow_and_update();242+ match snapshot.applied_resolution_revision {243+ Some(applied) if applied == revision => {244+ ensure!(245+ snapshot.applied_width == Some(width)246+ && snapshot.applied_height == Some(height),247+ "encoder resolution revision {revision} acknowledged {:?}x{:?}, expected {width}x{height}",248+ snapshot.applied_width,249+ snapshot.applied_height250+ );251+ return Ok(snapshot);252+ }253+ Some(applied) if applied > revision => bail!(254+ "encoder resolution revision {revision} was superseded by revision {applied}"255+ ),256+ _ => {}257+ }258+ if snapshot259+ .requested_resolution_revision260+ .is_some_and(|requested| requested > revision)261+ {262+ bail!(263+ "encoder resolution revision {revision} was coalesced before application"264+ );265+ }266+ status267+ .changed()268+ .await269+ .context("encoder control status closed before resolution was applied")?;270+ }271+ };272+ tokio::time::timeout(timeout, wait).await.with_context(|| {273+ format!("timed out waiting for encoder resolution revision {revision}")274+ })?275+ }189276 }190277 191278 @@ -210,6 +297,8 @@ impl EncoderControlTestHarness {
210297 211298 pub struct EncodedFrame {212299 pub sequence: u64,300+ pub width: u16,301+ pub height: u16,213302 /// Session-monotonic timestamp when the complete encoded access unit became214303 /// observable to the daemon, using the same epoch as audio. Raw PipeWire215304 /// capture PTS is not preserved by the current stdout bridge, so this must@@ -221,6 +310,9 @@ pub struct EncodedFrame {
221310 pub observed_at: Instant,222311 pub keyframe: bool,223312 pub codec_config: bool,313+ /// This configured keyframe begins a decoder generation that must not use314+ /// reference state from the preceding encoded resolution.315+ pub discontinuity: bool,224316 pub data: Arc<[u8]>,225317 }226318 @@ -294,6 +386,7 @@ async fn preflight_gamescope_static(config: &HostConfig) -> Result<()> {
294386 "videoconvert",295387 "videoscale",296388 "videorate",389+ "capsfilter",297390 pipewire.vaapi_encoder.as_str(),298391 "h264parse",299392 terminal_sink,@@ -419,6 +512,8 @@ fn spawn_gamescope_pipewire_in_process_with_target(
419512 max_gop_frames,420513 Some(preflight.pointer_surface_dimensions),421514 expected_device_path,515+ u16::try_from(config.width).context("configured width exceeds encoder metadata")?,516+ u16::try_from(config.height).context("configured height exceeds encoder metadata")?,422517 )423518 }424519 @@ -429,6 +524,8 @@ fn spawn_in_process_pipeline(
429524 max_gop_frames: usize,430525 pointer_surface_dimensions: Option<PointerSurfaceDimensions>,431526 expected_device_path: Option<String>,527+ initial_width: u16,528+ initial_height: u16,432529 ) -> Result<EncodedSource> {433530 let (sender, receiver) = watch::channel(None);434531 let (current_gop_sender, current_gop) = watch::channel(None);@@ -444,6 +541,8 @@ fn spawn_in_process_pipeline(
444541 desired,445542 status,446543 expected_device_path.as_deref(),544+ initial_width,545+ initial_height,447546 )448547 })449548 .await@@ -480,7 +579,8 @@ fn gamescope_in_process_pipeline_description(
480579 "pipewiresrc do-timestamp=true min-buffers=1 max-buffers=4 use-bufferpool=false target-object={target_object} \481580 ! queue max-size-buffers=1 max-size-bytes=0 max-size-time=0 leaky=downstream \482581 ! videorate drop-only=true max-rate={} \483- ! videoconvert ! videoscale ! {raw_caps} \582+ ! videoconvert ! videoscale \583+ ! capsfilter name=sigil_scale_caps caps={raw_caps} caps-change-mode=delayed \484584 ! {} name=sigil_encoder rate-control=cbr bitrate={bitrate} target-usage=7 key-int-max={} b-frames=0 ref-frames=1 aud=true \485585 ! h264parse config-interval=-1 \486586 ! video/x-h264,stream-format=byte-stream,alignment=au \@@ -545,6 +645,8 @@ async fn run_gamescope_pipewire(
545645 &sender,546646 ¤t_gop_sender,547647 interactive_gop_frames(config.framerate) as usize,648+ u16::try_from(config.width).context("configured width exceeds encoder metadata")?,649+ u16::try_from(config.height).context("configured height exceeds encoder metadata")?,548650 )549651 .await;550652 let _ = child.kill().await;@@ -625,6 +727,8 @@ async fn run_test_pattern(
625727 &sender,626728 ¤t_gop_sender,627729 interactive_gop_frames(config.framerate) as usize,730+ u16::try_from(config.width).context("configured width exceeds encoder metadata")?,731+ u16::try_from(config.height).context("configured height exceeds encoder metadata")?,628732 )629733 .await;630734 if result.is_ok() {@@ -644,12 +748,20 @@ async fn forward_annex_b_stream<R>(
644748 sender: &watch::Sender<Option<EncodedFrame>>,645749 current_gop_sender: &watch::Sender<Option<EncodedGop>>,646750 max_gop_frames: usize,751+ width: u16,752+ height: u16,647753 ) -> Result<()>648754 where649755 R: AsyncRead + Unpin,650756 {651- let mut publisher =652- AccessUnitPublisher::new(session_clock, sender, current_gop_sender, max_gop_frames);757+ let mut publisher = AccessUnitPublisher::new(758+ session_clock,759+ sender,760+ current_gop_sender,761+ max_gop_frames,762+ width,763+ height,764+ );653765 let mut parser = AnnexBAccessUnitParser::default();654766 let mut chunk = [0_u8; 16 * 1024];655767 @@ -677,26 +789,73 @@ struct AccessUnitPublisher<'a> {
677789 sender: &'a watch::Sender<Option<EncodedFrame>>,678790 current_gop_sender: &'a watch::Sender<Option<EncodedGop>>,679791 max_gop_frames: usize,792+ width: u16,793+ height: u16,680794 sequence: u64,681795 }682796 797+798+799+struct PendingResolutionTransition {800+ target: RevisedResolution,801+ deadline: Instant,802+}803+804+805+fn classify_resolution_sample(806+ current: (u16, u16),807+ pending: Option<PendingResolutionTransition>,808+ sample: (u16, u16),809+ configured_idr: bool,810+) -> Result<Option<bool>> {811+ if let Some(pending) = pending {812+ if sample != (pending.target.width, pending.target.height) || !configured_idr {813+ return Ok(None);814+ }815+ return Ok(Some(true));816+ }817+ ensure!(818+ sample == current,819+ "encoder changed resolution from {}x{} to {}x{} without a pending control revision",820+ current.0,821+ current.1,822+ sample.0,823+ sample.1824+ );825+ Ok(Some(false))826+}827+683828 impl<'a> AccessUnitPublisher<'a> {684829 fn new(685830 session_clock: SessionClock,686831 sender: &'a watch::Sender<Option<EncodedFrame>>,687832 current_gop_sender: &'a watch::Sender<Option<EncodedGop>>,688833 max_gop_frames: usize,834+ width: u16,835+ height: u16,689836 ) -> Self {690837 Self {691838 session_clock,692839 sender,693840 current_gop_sender,694841 max_gop_frames,842+ width,843+ height,695844 sequence: 0,696845 }697846 }698847 699848 fn publish(&mut self, access_unit: Vec<u8>) -> Result<bool> {849+ self.publish_with_metadata(access_unit, self.width, self.height, false)850+ }851+852+ fn publish_with_metadata(853+ &mut self,854+ access_unit: Vec<u8>,855+ width: u16,856+ height: u16,857+ discontinuity: bool,858+ ) -> Result<bool> {700859 ensure!(!access_unit.is_empty(), "encoded access unit is empty");701860 ensure!(702861 access_unit.len() <= MAX_ENCODE_BUFFER,@@ -710,11 +869,14 @@ impl<'a> AccessUnitPublisher<'a> {
710869 let codec_config = has_h264_codec_config(&access_unit);711870 let frame = EncodedFrame {712871 sequence: self.sequence,872+ width,873+ height,713874 capture_timestamp_micros,714875 presentation_timestamp_micros,715876 observed_at,716877 keyframe,717878 codec_config,879+ discontinuity,718880 data: Arc::from(access_unit),719881 };720882 self.sequence = self@@ -728,6 +890,8 @@ impl<'a> AccessUnitPublisher<'a> {
728890 self.max_gop_frames,729891 MAX_CURRENT_GOP_BYTES,730892 );893+ self.width = width;894+ self.height = height;731895 Ok(keyframe && codec_config)732896 }733897 @@ -746,6 +910,8 @@ fn run_in_process_pipeline(
746910 mut desired: watch::Receiver<EncoderControlDesired>,747911 status: watch::Sender<EncoderControlStatus>,748912 expected_device_path: Option<&str>,913+ initial_width: u16,914+ initial_height: u16,749915 ) -> Result<()> {750916 use gstreamer as gst;751917 use gstreamer::prelude::*;@@ -760,6 +926,9 @@ fn run_in_process_pipeline(
760926 let encoder = pipeline761927 .by_name("sigil_encoder")762928 .context("in-process GStreamer pipeline has no named encoder")?;929+ let scale_caps = pipeline930+ .by_name("sigil_scale_caps")931+ .context("in-process GStreamer pipeline has no named scale capsfilter")?;763932 let sink = pipeline764933 .by_name("sigil_sink")765934 .context("in-process GStreamer pipeline has no named appsink")?@@ -772,6 +941,7 @@ fn run_in_process_pipeline(
772941 sink.set_drop(false);773942 sink.set_sync(false);774943 validate_mutable_playing_bitrate(&encoder)?;944+ validate_resolution_capsfilter(&scale_caps)?;775945 776946 pipeline777947 .set_state(gst::State::Playing)@@ -789,10 +959,21 @@ fn run_in_process_pipeline(
789959 let bus = pipeline790960 .bus()791961 .context("in-process GStreamer pipeline has no bus")?;792- let mut publisher =793- AccessUnitPublisher::new(session_clock, &sender, ¤t_gop_sender, max_gop_frames);962+ let mut publisher = AccessUnitPublisher::new(963+ session_clock,964+ &sender,965+ ¤t_gop_sender,966+ max_gop_frames,967+ initial_width,968+ initial_height,969+ );794970 let mut applied_desired_revision = 0_u64;795971 let mut pending_force_keyframe_revision = None;972+ let mut pending_resolution = None;973+ status.send_modify(|status| {974+ status.applied_width = Some(initial_width);975+ status.applied_height = Some(initial_height);976+ });796977 797978 loop {798979 if let Some(message) = bus.timed_pop(gst::ClockTime::ZERO) {@@ -816,15 +997,32 @@ fn run_in_process_pipeline(
816997 if desired_state.revision > applied_desired_revision {817998 apply_encoder_control(818999 &encoder,1000+ &scale_caps,8191001 &sink,8201002 desired_state,8211003 &status,1004+ ¤t_gop_sender,8221005 &mut pending_force_keyframe_revision,1006+ &mut pending_resolution,8231007 )?;8241008 applied_desired_revision = desired_state.revision;8251009 }8261010 1011+ if pending_resolution.is_some_and(|pending: PendingResolutionTransition| {1012+ Instant::now() >= pending.deadline1013+ }) {1014+ let pending = pending_resolution.expect("checked as present");1015+ bail!(1016+ "encoder resolution revision {} did not produce a configured {}x{} IDR within {:?}",1017+ pending.target.revision,1018+ pending.target.width,1019+ pending.target.height,1020+ RESOLUTION_APPLY_TIMEOUT1021+ );1022+ }1023+8271024 if let Some(sample) = sink.try_pull_sample(gst::ClockTime::from_mseconds(50)) {1025+ let (width, height) = encoded_sample_dimensions(&sample)?;8281026 let buffer = sample8291027 .buffer()8301028 .context("in-process appsink sample has no buffer")?;@@ -835,7 +1033,34 @@ fn run_in_process_pipeline(
8351033 map.as_slice().len() <= MAX_ENCODE_BUFFER,8361034 "in-process encoded access unit exceeds {MAX_ENCODE_BUFFER} bytes"8371035 );838- let configured_idr = publisher.publish(map.as_slice().to_vec())?;1036+ let access_unit = map.as_slice();1037+ let configured_idr =1038+ is_h264_keyframe(access_unit) && has_h264_codec_config(access_unit);1039+ let Some(discontinuity) = classify_resolution_sample(1040+ (publisher.width, publisher.height),1041+ pending_resolution,1042+ (width, height),1043+ configured_idr,1044+ )?1045+ else {1046+ continue;1047+ };1048+ let configured_idr = publisher.publish_with_metadata(1049+ access_unit.to_vec(),1050+ width,1051+ height,1052+ discontinuity,1053+ )?;1054+ if discontinuity {1055+ let applied = pending_resolution1056+ .take()1057+ .expect("discontinuity requires a pending resolution");1058+ status.send_modify(|status| {1059+ status.applied_resolution_revision = Some(applied.target.revision);1060+ status.applied_width = Some(width);1061+ status.applied_height = Some(height);1062+ });1063+ }8391064 acknowledge_force_keyframe_if_configured_idr(8401065 configured_idr,8411066 &mut pending_force_keyframe_revision,@@ -899,18 +1124,102 @@ fn validate_mutable_playing_bitrate(encoder: &gstreamer::Element) -> Result<()>
8991124 Ok(())9001125 }9011126 1127+1128+fn validate_resolution_capsfilter(capsfilter: &gstreamer::Element) -> Result<()> {1129+ use gstreamer::prelude::*;1130+1131+ let property = capsfilter1132+ .find_property("caps")1133+ .context("configured scale capsfilter has no caps property")?;1134+ ensure!(1135+ property1136+ .flags()1137+ .contains(gstreamer::glib::ParamFlags::WRITABLE),1138+ "configured scale capsfilter caps property is not writable"1139+ );1140+ let caps = capsfilter.property::<gstreamer::Caps>("caps");1141+ let _ = dimensions_from_caps(&caps)?;1142+ Ok(())1143+}1144+1145+1146+fn dimensions_from_caps(caps: &gstreamer::CapsRef) -> Result<(u16, u16)> {1147+ let structure = caps1148+ .structure(0)1149+ .context("configured video caps have no structure")?;1150+ let width = structure1151+ .get::<i32>("width")1152+ .context("configured video caps have no integer width")?;1153+ let height = structure1154+ .get::<i32>("height")1155+ .context("configured video caps have no integer height")?;1156+ Ok((1157+ u16::try_from(width).context("configured video caps width is outside u16")?,1158+ u16::try_from(height).context("configured video caps height is outside u16")?,1159+ ))1160+}1161+1162+1163+fn encoded_sample_dimensions(sample: &gstreamer::Sample) -> Result<(u16, u16)> {1164+ let caps = sample1165+ .caps()1166+ .context("in-process encoded sample has no negotiated caps")?;1167+ dimensions_from_caps(caps)1168+}1169+1170+1171+fn apply_resolution_caps(1172+ capsfilter: &gstreamer::Element,1173+ resolution: RevisedResolution,1174+) -> Result<()> {1175+ use gstreamer::prelude::*;1176+1177+ let mut caps = capsfilter.property::<gstreamer::Caps>("caps");1178+ let caps_mut = caps.make_mut();1179+ let structure = caps_mut1180+ .structure_mut(0)1181+ .context("configured scale capsfilter has no caps structure")?;1182+ structure.set("width", i32::from(resolution.width));1183+ structure.set("height", i32::from(resolution.height));1184+ capsfilter.set_property("caps", caps);1185+ let readback = capsfilter.property::<gstreamer::Caps>("caps");1186+ ensure!(1187+ dimensions_from_caps(&readback)? == (resolution.width, resolution.height),1188+ "scale capsfilter resolution readback does not match requested {}x{}",1189+ resolution.width,1190+ resolution.height1191+ );1192+ Ok(())1193+}1194+1195+1196+fn push_configured_keyframe_request(sink: &gstreamer_app::AppSink) -> Result<()> {1197+ use gstreamer::prelude::*;1198+1199+ let event = gstreamer_video::UpstreamForceKeyUnitEvent::builder()1200+ .all_headers(true)1201+ .build();1202+ let sink_pad = sink1203+ .static_pad("sink")1204+ .context("configured in-process appsink has no sink pad")?;1205+ ensure!(1206+ sink_pad.push_event(event),1207+ "in-process encoder rejected upstream ForceKeyUnit"1208+ );1209+ Ok(())1210+}1211+9021212 9031213 fn apply_encoder_control(9041214 encoder: &gstreamer::Element,1215+ scale_caps: &gstreamer::Element,9051216 sink: &gstreamer_app::AppSink,9061217 desired: EncoderControlDesired,9071218 status: &watch::Sender<EncoderControlStatus>,1219+ current_gop_sender: &watch::Sender<Option<EncodedGop>>,9081220 pending_force_keyframe_revision: &mut Option<u64>,1221+ pending_resolution: &mut Option<PendingResolutionTransition>,9091222 ) -> Result<()> {910- use gstreamer as gst;911- use gstreamer::prelude::*;912- use gstreamer_video as gst_video;913-9141223 let current = *status.borrow();9151224 if let Some(bitrate) = desired.bitrate9161225 && current@@ -930,21 +1239,33 @@ fn apply_encoder_control(
9301239 });9311240 }9321241 1242+ let mut keyframe_requested = false;1243+ if let Some(resolution) = desired.resolution1244+ && current1245+ .requested_resolution_revision1246+ .is_none_or(|requested| resolution.revision > requested)1247+ {1248+ apply_resolution_caps(scale_caps, resolution)?;1249+ current_gop_sender.send_replace(None);1250+ push_configured_keyframe_request(sink)?;1251+ keyframe_requested = true;1252+ *pending_resolution = Some(PendingResolutionTransition {1253+ target: resolution,1254+ deadline: Instant::now() + RESOLUTION_APPLY_TIMEOUT,1255+ });1256+ status.send_modify(|status| {1257+ status.requested_resolution_revision = Some(resolution.revision);1258+ });1259+ }1260+9331261 if let Some(revision) = desired.force_keyframe_revision9341262 && current9351263 .requested_force_keyframe_revision9361264 .is_none_or(|requested| revision > requested)9371265 {938- let event = gst_video::UpstreamForceKeyUnitEvent::builder()939- .all_headers(true)940- .build();941- let sink_pad = sink942- .static_pad("sink")943- .context("configured in-process appsink has no sink pad")?;944- ensure!(945- sink_pad.push_event(event),946- "in-process encoder rejected upstream ForceKeyUnit"947- );1266+ if !keyframe_requested {1267+ push_configured_keyframe_request(sink)?;1268+ }9481269 *pending_force_keyframe_revision = Some(revision);9491270 status.send_modify(|status| {9501271 status.requested_force_keyframe_revision = Some(revision);@@ -1718,10 +2039,14 @@ mod tests {
17182039 17192040 let superseded_bitrate_revision = control.request_bitrate_kbps(4_000).unwrap();17202041 let bitrate_revision = control.request_bitrate_kbps(8_000).unwrap();2042+ let superseded_resolution_revision = control.request_resolution(960, 600).unwrap();2043+ let resolution_revision = control.request_resolution(640, 400).unwrap();17212044 let force_keyframe_revision = control.request_force_keyframe().unwrap();17222045 17232046 assert!(superseded_bitrate_revision < bitrate_revision);1724- assert!(bitrate_revision < force_keyframe_revision);2047+ assert!(bitrate_revision < superseded_resolution_revision);2048+ assert!(superseded_resolution_revision < resolution_revision);2049+ assert!(resolution_revision < force_keyframe_revision);17252050 let latest = *desired.borrow_and_update();17262051 assert_eq!(latest.revision, force_keyframe_revision);17272052 assert_eq!(@@ -1731,6 +2056,14 @@ mod tests {
17312056 kbps: 8_000,17322057 })17332058 );2059+ assert_eq!(2060+ latest.resolution,2061+ Some(RevisedResolution {2062+ revision: resolution_revision,2063+ width: 640,2064+ height: 400,2065+ })2066+ );17342067 assert_eq!(17352068 latest.force_keyframe_revision,17362069 Some(force_keyframe_revision)@@ -1740,6 +2073,10 @@ mod tests {
17402073 status.latest_revision = latest.revision;17412074 status.applied_bitrate_revision = Some(bitrate_revision);17422075 status.applied_bitrate_kbps = Some(8_000);2076+ status.requested_resolution_revision = Some(resolution_revision);2077+ status.applied_resolution_revision = Some(resolution_revision);2078+ status.applied_width = Some(640);2079+ status.applied_height = Some(400);17432080 status.requested_force_keyframe_revision = Some(force_keyframe_revision);17442081 });17452082 let mut pending = Some(force_keyframe_revision);@@ -1754,6 +2091,25 @@ mod tests {
17542091 .await17552092 .unwrap();17562093 assert_eq!(applied.applied_bitrate_kbps, Some(8_000));2094+ let applied_resolution = control2095+ .wait_for_resolution_applied(resolution_revision, 640, 400, Duration::from_millis(10))2096+ .await2097+ .unwrap();2098+ assert_eq!(applied_resolution.applied_width, Some(640));2099+ assert_eq!(applied_resolution.applied_height, Some(400));2100+ assert!(2101+ control2102+ .wait_for_resolution_applied(2103+ superseded_resolution_revision,2104+ 960,2105+ 600,2106+ Duration::from_millis(10),2107+ )2108+ .await2109+ .unwrap_err()2110+ .to_string()2111+ .contains("superseded")2112+ );17572113 let acknowledged = control17582114 .wait_for_force_keyframe_acknowledged(17592115 force_keyframe_revision,@@ -1766,6 +2122,8 @@ mod tests {
17662122 Some(force_keyframe_revision)17672123 );17682124 assert!(control.request_bitrate_kbps(0).is_err());2125+ assert!(control.request_resolution(0, 400).is_err());2126+ assert!(control.request_resolution(641, 400).is_err());17692127 assert!(17702128 control17712129 .request_bitrate_kbps(MAX_ENCODER_BITRATE_KBPS + 1)@@ -1790,16 +2148,17 @@ mod tests {
17902148 17912149 async fn in_process_gstreamer_x264_smoke() {17922150 let description = "videotestsrc is-live=true pattern=ball \1793- ! video/x-raw,width=320,height=180,framerate=30/1 \17942151 ! queue max-size-buffers=1 max-size-bytes=0 max-size-time=0 leaky=downstream \1795- ! videoconvert \2152+ ! videoconvert ! videoscale \2153+ ! capsfilter name=sigil_scale_caps caps=video/x-raw,width=320,height=180,framerate=30/1 caps-change-mode=delayed \17962154 ! x264enc name=sigil_encoder bitrate=2000 tune=zerolatency speed-preset=ultrafast key-int-max=15 bframes=0 byte-stream=true aud=true \17972155 ! h264parse config-interval=-1 \17982156 ! video/x-h264,stream-format=byte-stream,alignment=au \17992157 ! appsink name=sigil_sink max-buffers=1 drop=false sync=false"18002158 .to_owned();18012159 let mut source =1802- spawn_in_process_pipeline(description, SessionClock::start(), 15, None, None).unwrap();2160+ spawn_in_process_pipeline(description, SessionClock::start(), 15, None, None, 320, 180)2161+ .unwrap();18032162 let control = source18042163 .encoder_control18052164 .clone()@@ -1832,6 +2191,45 @@ mod tests {
18322191 .wait_for_force_keyframe_acknowledged(force_keyframe_revision, Duration::from_secs(10))18332192 .await18342193 .expect("x264 did not acknowledge a configured forced IDR");2194+ let before_resolution_sequence = source2195+ .current_gop2196+ .borrow()2197+ .as_ref()2198+ .and_then(|gop| gop.frames.first())2199+ .map(|frame| frame.sequence)2200+ .expect("x264 has no configured GOP before resolution transition");2201+2202+ let reduced_revision = control.request_resolution(256, 144).unwrap();2203+ control2204+ .wait_for_resolution_applied(reduced_revision, 256, 144, Duration::from_secs(10))2205+ .await2206+ .expect("x264 did not apply the reduced resolution");2207+ let reduced = source2208+ .current_gop2209+ .borrow()2210+ .as_ref()2211+ .and_then(|gop| gop.frames.first())2212+ .cloned()2213+ .expect("x264 has no reduced configured GOP");2214+ assert_eq!((reduced.width, reduced.height), (256, 144));2215+ assert!(reduced.keyframe && reduced.codec_config && reduced.discontinuity);2216+ assert!(reduced.sequence > before_resolution_sequence);2217+2218+ let restored_revision = control.request_resolution(320, 180).unwrap();2219+ control2220+ .wait_for_resolution_applied(restored_revision, 320, 180, Duration::from_secs(10))2221+ .await2222+ .expect("x264 did not restore the native resolution");2223+ let restored = source2224+ .current_gop2225+ .borrow()2226+ .as_ref()2227+ .and_then(|gop| gop.frames.first())2228+ .cloned()2229+ .expect("x264 has no restored configured GOP");2230+ assert_eq!((restored.width, restored.height), (320, 180));2231+ assert!(restored.keyframe && restored.codec_config && restored.discontinuity);2232+ assert!(restored.sequence > reduced.sequence);18352233 18362234 {18372235 let current_gop = source.current_gop.borrow();@@ -1867,6 +2265,9 @@ mod tests {
18672265 assert!(description.contains(18682266 "queue max-size-buffers=1 max-size-bytes=0 max-size-time=0 leaky=downstream"18692267 ));2268+ assert!(description.contains(2269+ "capsfilter name=sigil_scale_caps caps=video/x-raw,format=NV12,width=1280,height=800,framerate=60/1 caps-change-mode=delayed"2270+ ));18702271 assert!(18712272 description.contains("appsink name=sigil_sink max-buffers=1 drop=false sync=false")18722273 );@@ -1905,17 +2306,78 @@ mod tests {
19052306 assert!(!has_h264_codec_config(&[0, 0, 1, 8, 1, 2]));19062307 }19072308 2309+ 2310+ fn resolution_transition_suppresses_until_exact_configured_target_idr() {2311+ let transition = PendingResolutionTransition {2312+ target: RevisedResolution {2313+ revision: 7,2314+ width: 960,2315+ height: 600,2316+ },2317+ deadline: Instant::now() + Duration::from_secs(1),2318+ };2319+ assert!(transition.deadline > Instant::now());2320+ assert_eq!(2321+ classify_resolution_sample((1280, 800), Some(transition), (1280, 800), true).unwrap(),2322+ None2323+ );2324+ assert_eq!(2325+ classify_resolution_sample((1280, 800), Some(transition), (960, 600), false).unwrap(),2326+ None2327+ );2328+ assert_eq!(2329+ classify_resolution_sample((1280, 800), Some(transition), (960, 600), true).unwrap(),2330+ Some(true)2331+ );2332+ assert_eq!(2333+ classify_resolution_sample((960, 600), None, (960, 600), false).unwrap(),2334+ Some(false)2335+ );2336+ assert!(classify_resolution_sample((960, 600), None, (1280, 800), true).is_err());2337+ }2338+2339+ 2340+ fn publisher_carries_exact_resolution_and_discontinuity_into_new_gop() {2341+ let (frame_sender, frame_receiver) = watch::channel(None);2342+ let (current_gop_sender, current_gop_receiver) = watch::channel(None);2343+ let mut publisher = AccessUnitPublisher::new(2344+ SessionClock::start(),2345+ &frame_sender,2346+ ¤t_gop_sender,2347+ 30,2348+ 1280,2349+ 800,2350+ );2351+ let configured_idr = vec![0, 0, 0, 1, 7, 1, 0, 0, 0, 1, 8, 1, 0, 0, 0, 1, 5, 1];2352+2353+ assert!(2354+ publisher2355+ .publish_with_metadata(configured_idr, 960, 600, true)2356+ .unwrap()2357+ );2358+ let frame = frame_receiver.borrow().as_ref().unwrap().clone();2359+ assert_eq!((frame.width, frame.height), (960, 600));2360+ assert!(frame.discontinuity);2361+ let current_gop = current_gop_receiver.borrow();2362+ let first = ¤t_gop.as_ref().unwrap().frames[0];2363+ assert_eq!((first.width, first.height), (960, 600));2364+ assert!(first.keyframe && first.codec_config && first.discontinuity);2365+ }2366+19082367 19092368 fn current_gop_retains_only_a_bounded_contiguous_decodable_chain() {19102369 let (frame_sender, frame_receiver) = watch::channel(None);19112370 let (current_gop_sender, current_gop_receiver) = watch::channel(None);19122371 let frame = |sequence, keyframe, codec_config, payload_len| EncodedFrame {19132372 sequence,2373+ width: 1280,2374+ height: 800,19142375 capture_timestamp_micros: sequence,19152376 presentation_timestamp_micros: sequence as i64,19162377 observed_at: Instant::now(),19172378 keyframe,19182379 codec_config,2380+ discontinuity: false,19192381 data: Arc::from(vec![sequence as u8; payload_len]),19202382 };19212383 @@ -1289,6 +1289,38 @@ buffer metadata through an in-process appsink before making those stronger
12891289 claims; until then, correlate the probe with Gamescope/PipeWire statistics and12901290 the external sampling in the evidence section.12911291 1292+### Validate motion-sensitive resolution1293+1294+This gate applies only to a Linux build with `in-process-gstreamer` and a host1295+configuration whose `encoder_backend` is `in-process-gstreamer`. The external1296+`gst-launch` compatibility backend intentionally remains fixed-resolution.1297+1298+Connect Portal, enable host debug logs, and repeat at least ten cycles of a1299+high-motion 3D scene followed by a static high-frequency test card. For each1300+cycle retain the host journal and Portal console. The host decision record must1301+show an applied 960x600 motion/pressure target, followed by an applied 1280x8001302+recovery target only after at least two seconds of stillness, three fresh clean1303+feedback windows, and the three-second transition cooldown. A stale feedback1304+window must never restore native resolution.1305+1306+At each boundary verify that the first delivered target-size frame is a1307+keyframe with SPS/PPS and a discontinuity, no preceding-resolution delta is1308+presented afterward, and Portal stays connected without a WebCodecs error.1309+Absolute pointer tests at the canvas center and four corners must continue to1310+land on the native Gamescope surface at both encoded resolutions. The Portal1311+window must not resize merely because 1280x800 and 960x600 have the same aspect1312+ratio.1313+1314+Record transition request-to-configured-IDR time and Portal1315+configured-IDR-to-first-presentation time separately. These are recovery1316+measurements, not glass-to-glass latency. Require ten of ten successful1317+downscale/restore cycles, recovery p95 no greater than 900 ms and no individual1318+recovery greater than two seconds, video/decode queue p95 no greater than two,1319+and no sustained latency growth. Measure true glass-to-glass latency with a1320+paired high-speed-camera run comparing fixed-native and adaptive sessions under1321+the same controlled bandwidth pressure; do not substitute post-encode queue1322+age for that result.1323+12921324 Then perform the actual headless gate:12931325 12941326 1. Shut the host down.@@ -5,9 +5,10 @@ export const MAX_FRAME_PIXELS = 7680 * 4320;
55 66 const MAGIC = [0x53, 0x47, 0x46, 0x52]; // SGFR77 const VERSION = 1;8-const KNOWN_FLAGS = 0b00000011;8+const KNOWN_FLAGS = 0b00000111;99 const KEYFRAME_FLAG = 1 << 0;1010 const DISCONTINUITY_FLAG = 1 << 1;11+const CODEC_CONFIG_FLAG = 1 << 2;1112 const OPTIONAL_U64_NONE = 0xffffffffffffffffn;1213 const OPTIONAL_I64_NONE = -0x8000000000000000n;1314 const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER);@@ -53,6 +54,9 @@ export function parseFrameEnvelope(message) {
5354 if (!codec) throw new Error(`unsupported frame channel codec: ${bytes[5]}`);5455 const flags = bytes[6];5556 if ((flags & ~KNOWN_FLAGS) !== 0) throw new Error('frame channel has unknown flag bits');57+ if ((flags & CODEC_CONFIG_FLAG) !== 0 && (flags & KEYFRAME_FLAG) === 0) {58+ throw new Error('frame channel codec configuration requires a keyframe');59+ }5660 if (bytes[7] !== 0) throw new Error('frame channel reserved byte must be zero');5761 5862 const width = view.getUint16(8, false);@@ -89,6 +93,7 @@ export function parseFrameEnvelope(message) {
8993 codec,9094 keyframe: (flags & KEYFRAME_FLAG) !== 0,9195 discontinuity: (flags & DISCONTINUITY_FLAG) !== 0,96+ codecConfig: (flags & CODEC_CONFIG_FLAG) !== 0,9297 sequence,9398 captureTimestampMicros,9499 ptsMicros,@@ -9,7 +9,7 @@ import {
99 1010 function frameEnvelope({1111 codec = 1,12- flags = 3,12+ flags = 7,1313 width = 1280,1414 height = 800,1515 payload = Uint8Array.of(0, 0, 0, 1, 0x65),@@ -42,6 +42,7 @@ test('parses an exact v1 frame envelope without copying its payload', () => {
4242 codec: frame.codec,4343 keyframe: frame.keyframe,4444 discontinuity: frame.discontinuity,45+ codecConfig: frame.codecConfig,4546 sequence: frame.sequence,4647 captureTimestampMicros: frame.captureTimestampMicros,4748 ptsMicros: frame.ptsMicros,@@ -52,6 +53,7 @@ test('parses an exact v1 frame envelope without copying its payload', () => {
5253 codec: 'h264',5354 keyframe: true,5455 discontinuity: true,56+ codecConfig: true,5557 sequence: 42,5658 captureTimestampMicros: 123456,5759 ptsMicros: 98765,@@ -79,6 +81,7 @@ test('rejects malformed identity, version, codec, flags, and reserved fields', (
7981 (bytes) => { bytes[4] = 2; },8082 (bytes) => { bytes[5] = 0; },8183 (bytes) => { bytes[6] = 0x80; },84+ (bytes) => { bytes[6] = 0b100; },8285 (bytes) => { bytes[7] = 1; },8386 ];8487 for (const mutate of mutations) {@@ -51,9 +51,9 @@ export function resolvePointerSurfaceSize(
5151 pointerSurfaceDimensions,5252 frameWidth,5353 frameHeight,54- relativePointer,54+ _relativePointer,5555 ) {56- if (relativePointer && pointerSurfaceDimensions !== null) {56+ if (pointerSurfaceDimensions !== null) {5757 return pointerSurfaceDimensions;5858 }5959 if (@@ -65,6 +65,29 @@ export function resolvePointerSurfaceSize(
6565 return { width: frameWidth, height: frameHeight };6666 }6767 68+export function mapCanvasPointToSurface({ clientX, clientY, rect, surface }) {69+ if (70+ !Number.isFinite(clientX)71+ || !Number.isFinite(clientY)72+ || !Number.isFinite(rect?.left)73+ || !Number.isFinite(rect?.top)74+ || !Number.isFinite(rect?.width)75+ || rect.width <= 076+ || !Number.isFinite(rect?.height)77+ || rect.height <= 078+ || !Number.isSafeInteger(surface?.width)79+ || surface.width <= 080+ || !Number.isSafeInteger(surface?.height)81+ || surface.height <= 082+ ) return null;83+ return {84+ x: Math.max(0, Math.min(surface.width - 1,85+ Math.floor((clientX - rect.left) * surface.width / rect.width))),86+ y: Math.max(0, Math.min(surface.height - 1,87+ Math.floor((clientY - rect.top) * surface.height / rect.height))),88+ };89+}90+6891 export function validatePointerPositionFeedback(value, surface = null) {6992 if (!Number.isSafeInteger(value?.sequence) || value.sequence < 0) {7093 throw new TypeError('invalid pointer feedback sequence');@@ -10,6 +10,7 @@ import {
1010 RelativePointerAccumulator,1111 advanceRemotePointerPosition,1212 browserPointerLockLossRequiresControlExit,13+ mapCanvasPointToSurface,1314 resolvePointerSurfaceSize,1415 restoreRejectedPointerMotion,1516 scaleRelativePointerDelta,@@ -61,7 +62,7 @@ test('pointer surface metadata is bounded and old hosts fall back to frame size'
6162 );6263 assert.deepEqual(6364 resolvePointerSurfaceSize(native, 1280, 800, false),64- { width: 1280, height: 800 },65+ { width: 2560, height: 1600 },6566 );6667 assert.equal(resolvePointerSurfaceSize(null, 0, 800, true), null);6768 assert.throws(@@ -74,6 +75,19 @@ test('pointer surface metadata is bounded and old hosts fall back to frame size'
7475 );7576 });7677 78+test('absolute pointer mapping always targets the negotiated native surface', () => {79+ const rect = { left: 20, top: 10, width: 640, height: 400 };80+ const surface = { width: 1280, height: 800 };81+ assert.deepEqual(mapCanvasPointToSurface({ clientX: 340, clientY: 210, rect, surface }), {82+ x: 640,83+ y: 400,84+ });85+ assert.deepEqual(mapCanvasPointToSurface({ clientX: 700, clientY: -10, rect, surface }), {86+ x: 1279,87+ y: 0,88+ });89+});90+7791 test('host pointer feedback is bounded and validated against the native surface', () => {7892 const surface = { width: 2560, height: 1600 };7993 assert.deepEqual(@@ -17,6 +17,7 @@ import {
1717 MAX_HELD_MOUSE_BUTTONS,1818 PointerMotionBuffer,1919 browserPointerLockLossRequiresControlExit,20+ mapCanvasPointToSurface,2021 resolvePointerSurfaceSize,2122 restoreRejectedPointerMotion,2223 scaleRelativePointerDelta,@@ -86,7 +87,9 @@ import {
8687 constrainStreamWindowResize,8788 fitInitialStreamWindow,8889 fitStreamSurface,90+ streamAspectKey,8991 } from './window-geometry.mjs';92+import { VideoFormatTransitionGuard } from './video-format-transition.mjs';9093 import { buildVideoDecoderConfig } from './video-decoder-config.mjs';9194 import {9295 DECODER_RECOVERY_REASONS,@@ -123,7 +126,8 @@ let inputCapabilities = {
123126 };124127 let frameWidth = 0;125128 let frameHeight = 0;126-let fittedStreamDimensions = null;129+let fittedStreamAspect = null;130+let pendingStreamFit = null;127131 let lastObservedWindowSize = null;128132 let streamWindowResizeTimer = null;129133 let pointerSurfaceDimensions = null;@@ -132,6 +136,7 @@ let remotePointerVisible = false;
132136 let activeFrameChannel = null;133137 let activeFrameGeneration = null;134138 let activeFrameSession = null;139+let activeVideoFormatEpoch = 0;135140 let activeAudioChannel = null;136141 let activePointerChannel = null;137142 let activePointerSession = null;@@ -1001,7 +1006,8 @@ async function disconnect() {
10011006 control: false,10021007 };10031008 pointerSurfaceDimensions = null;1004- fittedStreamDimensions = null;1009+ fittedStreamAspect = null;1010+ pendingStreamFit = null;10051011 lastObservedWindowSize = null;10061012 if (streamWindowResizeTimer !== null) clearTimeout(streamWindowResizeTimer);10071013 streamWindowResizeTimer = null;@@ -1122,6 +1128,7 @@ function requestDecoderKeyframe(reason) {
11221128 const decoderRecovery = new DecoderRecoveryState({11231129 onKeyframeRequest: requestDecoderKeyframe,11241130 });1131+const videoFormatTransition = new VideoFormatTransitionGuard();11251132 11261133 function enterDecoderRecovery(reason, { restart = false, requestKeyframe = true } = {}) {11271134 const result = restart@@ -1231,6 +1238,8 @@ function teardownDecoderPipeline() {
12311238 function resetStreamTelemetry() {12321239 teardownDecoderPipeline();12331240 decoderRecovery.reset();1241+ videoFormatTransition.reset();1242+ activeVideoFormatEpoch = 0;12341243 droppedFrames = 0;12351244 transportDroppedFrames = 0;12361245 transportObjectDroppedFrames = null;@@ -1492,8 +1501,10 @@ async function applyClientWindowGeometry(geometry, unmaximize) {
14921501 14931502 async function fitWindowToIncomingStream() {14941503 if (!connected || frameWidth < 1 || frameHeight < 1) return false;1495- const dimensions = `${frameWidth}x${frameHeight}`;1496- if (fittedStreamDimensions === dimensions) return false;1504+ const aspect = streamAspectKey(frameWidth, frameHeight);1505+ if (fittedStreamAspect === aspect || pendingStreamFit?.aspect === aspect) return false;1506+ const request = { aspect, generation: activeFrameGeneration, epoch: activeVideoFormatEpoch };1507+ pendingStreamFit = request;14971508 const geometry = fitInitialStreamWindow({14981509 frameWidth,14991510 frameHeight,@@ -1502,8 +1513,17 @@ async function fitWindowToIncomingStream() {
15021513 availableHeight: window.screen.availHeight,15031514 });15041515 const applied = await applyClientWindowGeometry(geometry, true);1505- if (applied) fittedStreamDimensions = dimensions;1506- return applied;1516+ const currentAspect = frameWidth > 0 && frameHeight > 01517+ ? streamAspectKey(frameWidth, frameHeight)1518+ : null;1519+ const current = pendingStreamFit === request1520+ && connected1521+ && activeFrameGeneration === request.generation1522+ && activeVideoFormatEpoch === request.epoch1523+ && currentAspect === request.aspect;1524+ if (pendingStreamFit === request) pendingStreamFit = null;1525+ if (applied && current) fittedStreamAspect = request.aspect;1526+ return applied && current;15071527 }15081528 15091529 function scheduleStreamWindowAspectCorrection() {@@ -1534,27 +1554,49 @@ function scheduleStreamWindowAspectCorrection() {
15341554 }, 80);15351555 }15361556 1557+function commitVideoFrame(plan, keyframe, enqueued) {1558+ if (!enqueued) {1559+ finishVideoChunkEnqueue(keyframe, false);1560+ return false;1561+ }1562+ let committed;1563+ try {1564+ committed = videoFormatTransition.commit(plan);1565+ } catch (error) {1566+ console.warn('discarding stale video format transaction:', error);1567+ enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET);1568+ return false;1569+ }1570+ if (plan.reconfigure) {1571+ activeCodec = committed.format.codec;1572+ frameWidth = committed.format.width;1573+ frameHeight = committed.format.height;1574+ activeVideoFormatEpoch = committed.epoch;1575+ if (canvas.width !== frameWidth) canvas.width = frameWidth;1576+ if (canvas.height !== frameHeight) canvas.height = frameHeight;1577+ sizeCanvasToIncomingStream();1578+ if (connected) void fitWindowToIncomingStream();1579+ }1580+ finishVideoChunkEnqueue(keyframe, true);1581+ return true;1582+}1583+15371584 function processFramePayload(payload, binaryData = null) {15381585 const receivedAt = performance.now();15391586 frontendDeliveryRate.record(receivedAt);15401587 frontendDeliveryCadence.record(receivedAt);1541- const { width, height, data, codec, keyframe, pts_micros: ptsMicros, discontinuity } = payload;1588+ const {1589+ width,1590+ height,1591+ data,1592+ codec = 'h264',1593+ keyframe,1594+ codecConfig = keyframe,1595+ sequence = null,1596+ pts_micros: ptsMicros,1597+ discontinuity = false,1598+ } = payload;15421599 if (discontinuity) resetAvSyncTelemetry();1543- const dimensionsChanged = frameWidth !== width || frameHeight !== height;1544- frameWidth = width;1545- frameHeight = height;1546- if (canvas.width !== width) canvas.width = width;1547- if (canvas.height !== height) canvas.height = height;1548- if (dimensionsChanged) sizeCanvasToIncomingStream();1549- if (dimensionsChanged && connected) void fitWindowToIncomingStream();1550-1551- if (codec && codec !== activeCodec) {1552- activeCodec = codec;1553- decoderConfigured = false;1554- enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET);1555- console.log('codec changed:', activeCodec);1556- }1557-15581600 receivedFrames++;15591601 15601602 if (hasWebCodecs) {@@ -1562,14 +1604,48 @@ function processFramePayload(payload, binaryData = null) {
15621604 // base64 branch only supports an older in-process sender during migration.15631605 const raw = binaryData ?? Uint8Array.from(atob(data), c => c.charCodeAt(0));15641606 const decoderBacklogged = videoDecoder?.decodeQueueSize >= MAX_DECODE_QUEUE_SIZE;1565- if (discontinuity) {1607+ if (decoderBacklogged) {1608+ enterDecoderRecovery(DECODER_RECOVERY_REASONS.FRONTEND_BACKPRESSURE);1609+ }1610+ const decoderBoundary = discontinuity || !decoderConfigured;1611+ let formatPlan;1612+ try {1613+ formatPlan = videoFormatTransition.plan({1614+ sequence,1615+ codec,1616+ width,1617+ height,1618+ keyframe,1619+ codecConfig,1620+ discontinuity: decoderBoundary,1621+ });1622+ } catch (error) {1623+ console.warn('invalid video format transition:', error);1624+ droppedFrames++;1625+ enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET);1626+ return;1627+ }1628+ if (formatPlan.action === 'drop-stale') {1629+ droppedFrames++;1630+ return;1631+ }1632+ if (formatPlan.action === 'recover') {1633+ droppedFrames++;1634+ enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET);1635+ return;1636+ }1637+ if (formatPlan.reconfigure) {15661638 // A delivered discontinuity keyframe is the recovery boundary we were15671639 // waiting for, not evidence that another keyframe is missing.1568- enterDecoderRecovery(DECODER_RECOVERY_REASONS.DISCONTINUITY, {1569- requestKeyframe: !keyframe,1570- });1571- } else if (decoderBacklogged) {1572- enterDecoderRecovery(DECODER_RECOVERY_REASONS.FRONTEND_BACKPRESSURE);1640+ if (discontinuity) {1641+ enterDecoderRecovery(DECODER_RECOVERY_REASONS.DISCONTINUITY, {1642+ requestKeyframe: !keyframe,1643+ });1644+ } else {1645+ enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET, {1646+ requestKeyframe: false,1647+ });1648+ }15731649 }15741650 if (decoderRecovery.shouldDropFrame({ keyframe })) {15751651 if (!decoderRecovery.requestIssued) {@@ -1583,12 +1659,12 @@ function processFramePayload(payload, binaryData = null) {
15831659 ? mediaPtsMicros15841660 : performance.now() * 1000;15851661 1586- if (activeCodec === 'av1') {1662+ if (codec === 'av1') {15871663 // ── AV1 path ──15881664 const obus = parseAv1Obus(raw);15891665 if (keyframe) {15901666 const seqHeader = obus.find(o => o.type === 12);1591- if (seqHeader && !decoderConfigured) {1667+ if (seqHeader && formatPlan.reconfigure) {15921668 initWebCodecsDecoder(width, height, buildAv1Description(seqHeader.data), av1CodecStr(seqHeader.data));15931669 }15941670 }@@ -1606,10 +1682,10 @@ function processFramePayload(payload, binaryData = null) {
16061682 data: chunkData,16071683 });16081684 const enqueued = enqueueVideoChunk(chunk, receivedAt, 'av1', mediaPtsMicros);1609- finishVideoChunkEnqueue(keyframe, enqueued);1685+ commitVideoFrame(formatPlan, keyframe, enqueued);16101686 }16111687 1612- } else if (activeCodec === 'h265') {1688+ } else if (codec === 'h265') {16131689 // ── H.265 path ──16141690 const nals = parseAnnexBNals(raw);16151691 if (keyframe) {@@ -1620,7 +1696,7 @@ function processFramePayload(payload, binaryData = null) {
16201696 else if (t === 33) sps = nal;16211697 else if (t === 34) pps = nal;16221698 }1623- if (vps && sps && pps && !decoderConfigured) {1699+ if (vps && sps && pps && formatPlan.reconfigure) {16241700 initWebCodecsDecoder(width, height, buildHvcDescription(vps, sps, pps), hevcCodecStr(sps));16251701 }16261702 }@@ -1636,7 +1712,7 @@ function processFramePayload(payload, binaryData = null) {
16361712 data: nalsToLengthPrefixed(sliceNals),16371713 });16381714 const enqueued = enqueueVideoChunk(chunk, receivedAt, 'hevc', mediaPtsMicros);1639- finishVideoChunkEnqueue(keyframe, enqueued);1715+ commitVideoFrame(formatPlan, keyframe, enqueued);16401716 }16411717 16421718 } else {@@ -1649,7 +1725,7 @@ function processFramePayload(payload, binaryData = null) {
16491725 if (t === 7) sps = nal;16501726 else if (t === 8) pps = nal;16511727 }1652- if (sps && pps && !decoderConfigured) {1728+ if (sps && pps && formatPlan.reconfigure) {16531729 initWebCodecsDecoder(width, height, buildAvcDescription(sps, pps), avcCodecStr(sps));16541730 }16551731 }@@ -1665,11 +1741,18 @@ function processFramePayload(payload, binaryData = null) {
16651741 data: nalsToLengthPrefixed(sliceNals),16661742 });16671743 const enqueued = enqueueVideoChunk(chunk, receivedAt, 'h264', mediaPtsMicros);1668- finishVideoChunkEnqueue(keyframe, enqueued);1744+ commitVideoFrame(formatPlan, keyframe, enqueued);16691745 }16701746 }16711747 } else {16721748 // JPEG fallback (openh264 decode → JPEG encode in Rust)1749+ const dimensionsChanged = frameWidth !== width || frameHeight !== height;1750+ frameWidth = width;1751+ frameHeight = height;1752+ if (canvas.width !== width) canvas.width = width;1753+ if (canvas.height !== height) canvas.height = height;1754+ if (dimensionsChanged) sizeCanvasToIncomingStream();1755+ if (dimensionsChanged && connected) void fitWindowToIncomingStream();16731756 const bytes = Uint8Array.from(atob(data), c => c.charCodeAt(0));16741757 const blob = new Blob([bytes], { type: 'image/jpeg' });16751758 const url = URL.createObjectURL(blob);@@ -1745,6 +1828,8 @@ function handleBinaryFrameMessage(message, session) {
17451828 data: null,17461829 codec: frame.codec,17471830 keyframe: frame.keyframe,1831+ codecConfig: frame.codecConfig,1832+ sequence: frame.sequence,17481833 pts_micros: frame.ptsMicros,17491834 discontinuity: frame.discontinuity,17501835 }, frame.data);@@ -2214,12 +2299,12 @@ function updateControlUI() {
22142299 22152300 function scaleCoords(clientX, clientY) {22162301 const rect = canvas.getBoundingClientRect();2217- const scaleX = frameWidth / rect.width;2218- const scaleY = frameHeight / rect.height;2219- return {2220- x: Math.round((clientX - rect.left) * scaleX),2221- y: Math.round((clientY - rect.top) * scaleY),2222- };2302+ return mapCanvasPointToSurface({2303+ clientX,2304+ clientY,2305+ rect,2306+ surface: currentPointerSurfaceSize(),2307+ }) ?? { x: 0, y: 0 };22232308 }22242309 22252310 function mapKey(e) {@@ -0,0 +1,68 @@
1+function validFormat({ codec, width, height }) {2+ if (!['h264', 'h265', 'av1'].includes(codec)) throw new TypeError('invalid video codec');3+ if (!Number.isSafeInteger(width) || width <= 0 || !Number.isSafeInteger(height) || height <= 0) {4+ throw new TypeError('invalid video dimensions');5+ }6+ return Object.freeze({ codec, width, height });7+}8+9+function sameFormat(left, right) {10+ return left !== null11+ && left.codec === right.codec12+ && left.width === right.width13+ && left.height === right.height;14+}15+16+export class VideoFormatTransitionGuard {17+ #format = null;18+ #lastSequence = null;19+ #epoch = 0;20+ #revision = 0;21+22+ get format() { return this.#format; }23+ get epoch() { return this.#epoch; }24+25+ reset() {26+ this.#format = null;27+ this.#lastSequence = null;28+ this.#epoch = 0;29+ this.#revision++;30+ }31+32+ plan({ sequence, codec, width, height, keyframe, codecConfig, discontinuity }) {33+ const format = validFormat({ codec, width, height });34+ if (sequence !== null && sequence !== undefined35+ && (!Number.isSafeInteger(sequence) || sequence < 0)) {36+ throw new TypeError('invalid video sequence');37+ }38+ if (codecConfig && !keyframe) throw new Error('codec configuration requires a keyframe');39+ if (sequence !== null && sequence !== undefined40+ && this.#lastSequence !== null && sequence <= this.#lastSequence) {41+ return Object.freeze({ action: 'drop-stale', epoch: this.#epoch });42+ }43+44+ const reconfigure = !sameFormat(this.#format, format) || discontinuity;45+ if (reconfigure && !(keyframe && codecConfig)) {46+ return Object.freeze({ action: 'recover', epoch: this.#epoch });47+ }48+ return Object.freeze({49+ action: 'accept',50+ reconfigure,51+ format,52+ sequence: sequence ?? null,53+ epoch: this.#epoch + (reconfigure ? 1 : 0),54+ revision: this.#revision,55+ });56+ }57+58+ commit(plan) {59+ if (plan?.action !== 'accept' || plan.revision !== this.#revision) {60+ throw new Error('stale video format transition');61+ }62+ if (plan.reconfigure) this.#format = plan.format;63+ if (plan.sequence !== null) this.#lastSequence = plan.sequence;64+ this.#epoch = plan.epoch;65+ this.#revision++;66+ return Object.freeze({ format: this.#format, epoch: this.#epoch });67+ }68+}@@ -0,0 +1,57 @@
1+import assert from 'node:assert/strict';2+import test from 'node:test';3+4+import { VideoFormatTransitionGuard } from './video-format-transition.mjs';5+6+const frame = (overrides = {}) => ({7+ sequence: 1,8+ codec: 'h264',9+ width: 1280,10+ height: 800,11+ keyframe: true,12+ codecConfig: true,13+ discontinuity: false,14+ ...overrides,15+});16+17+test('format transitions require configured keyframes and advance epochs monotonically', () => {18+ const guard = new VideoFormatTransitionGuard();19+ assert.equal(guard.plan(frame({ keyframe: false, codecConfig: false })).action, 'recover');20+21+ const initial = guard.plan(frame());22+ assert.equal(initial.reconfigure, true);23+ assert.equal(initial.epoch, 1);24+ guard.commit(initial);25+26+ const ordinary = guard.plan(frame({ sequence: 2, keyframe: false, codecConfig: false }));27+ assert.equal(ordinary.reconfigure, false);28+ assert.equal(ordinary.epoch, 1);29+ guard.commit(ordinary);30+31+ const resized = guard.plan(frame({ sequence: 3, width: 640, height: 400 }));32+ assert.equal(resized.reconfigure, true);33+ assert.equal(resized.epoch, 2);34+ guard.commit(resized);35+ assert.deepEqual(guard.format, { codec: 'h264', width: 640, height: 400 });36+});37+38+test('stale outputs and unconfigured format changes cannot commit', () => {39+ const guard = new VideoFormatTransitionGuard();40+ guard.commit(guard.plan(frame()));41+ assert.equal(guard.plan(frame({ sequence: 1 })).action, 'drop-stale');42+ assert.equal(guard.plan(frame({ sequence: 2, width: 640, height: 400,43+ keyframe: false, codecConfig: false })).action, 'recover');44+45+ const first = guard.plan(frame({ sequence: 2, keyframe: false, codecConfig: false }));46+ const competing = guard.plan(frame({ sequence: 3, keyframe: false, codecConfig: false }));47+ guard.commit(first);48+ assert.throws(() => guard.commit(competing), /stale video format transition/);49+});50+51+test('a discontinuity starts a new configured epoch even at the same dimensions', () => {52+ const guard = new VideoFormatTransitionGuard();53+ guard.commit(guard.plan(frame()));54+ const boundary = guard.plan(frame({ sequence: 2, discontinuity: true }));55+ assert.equal(boundary.reconfigure, true);56+ assert.equal(boundary.epoch, 2);57+});@@ -24,6 +24,18 @@ function roundedGeometry(width, height) {
2424 });2525 }2626 27+export function streamAspectKey(frameWidth, frameHeight) {28+ const width = positiveFinite(frameWidth, 'frame width');29+ const height = positiveFinite(frameHeight, 'frame height');30+ if (!Number.isSafeInteger(width) || !Number.isSafeInteger(height)) {31+ throw new RangeError('stream dimensions must be exact integers');32+ }33+ let a = width;34+ let b = height;35+ while (b !== 0) [a, b] = [b, a % b];36+ return `${width / a}:${height / a}`;37+}38+2739 export function fitInitialStreamWindow({2840 frameWidth,2941 frameHeight,@@ -5,8 +5,15 @@ import {
55 constrainStreamWindowResize,66 fitInitialStreamWindow,77 fitStreamSurface,8+ streamAspectKey,89 } from './window-geometry.mjs';910 11+test('equivalent stream resolutions share one stable aspect identity', () => {12+ assert.equal(streamAspectKey(1280, 800), '8:5');13+ assert.equal(streamAspectKey(640, 400), '8:5');14+ assert.notEqual(streamAspectKey(1280, 720), streamAspectKey(1280, 800));15+});16+1017 test('initial stream window fits the display while preserving the drawable aspect', () => {1118 const geometry = fitInitialStreamWindow({1219 frameWidth: 1280,@@ -42,7 +42,7 @@ if [[ "${GOQ_VERIFY_IN_PROCESS_GSTREAMER:-0}" == 1 ]]; then
4242 echo 'in-process GStreamer gate requires gst-inspect-1.0' >&24343 exit 14444 }45- for gstreamer_element in videotestsrc queue videoconvert x264enc h264parse appsink; do45+ for gstreamer_element in videotestsrc queue videoconvert videoscale capsfilter x264enc h264parse appsink; do4646 gst-inspect-1.0 "$gstreamer_element" >/dev/null || {4747 printf 'in-process GStreamer gate requires the %s plugin\n' \4848 "$gstreamer_element" >&2@@ -109,6 +109,7 @@ const FRAME_CHANNEL_VERSION: u8 = 1;
109109 const FRAME_CHANNEL_HEADER_LEN: usize = 40;110110 const FRAME_CHANNEL_FLAG_KEYFRAME: u8 = 1 << 0;111111 const FRAME_CHANNEL_FLAG_DISCONTINUITY: u8 = 1 << 1;112+const FRAME_CHANNEL_FLAG_CODEC_CONFIG: u8 = 1 << 2;112113 const FRAME_CHANNEL_OPTIONAL_U64_NONE: u64 = u64::MAX;113114 const FRAME_CHANNEL_OPTIONAL_I64_NONE: i64 = i64::MIN;114115 const AUDIO_CHANNEL_MAGIC: [u8; 4] = *b"SGAC";@@ -572,6 +573,7 @@ struct FrameEnvelopeMetadata<'a> {
572573 codec: &'a str,573574 keyframe: bool,574575 discontinuity: bool,576+ codec_config: bool,575577 sequence: Option<u64>,576578 capture_timestamp_micros: Option<u64>,577579 pts_micros: Option<i64>,@@ -582,6 +584,9 @@ fn encode_frame_envelope(
582584 payload: &[u8],583585 ) -> Result<Vec<u8>, String> {584586 validate_legacy_media_header(metadata.width, metadata.height, payload.len())?;587+ if metadata.codec_config && !metadata.keyframe {588+ return Err("Frame codec configuration requires a keyframe".to_string());589+ }585590 if metadata.sequence == Some(FRAME_CHANNEL_OPTIONAL_U64_NONE) {586591 return Err("Frame sequence collides with the channel sentinel".to_string());587592 }@@ -622,6 +627,9 @@ fn encode_frame_envelope(
622627 if metadata.discontinuity {623628 flags |= FRAME_CHANNEL_FLAG_DISCONTINUITY;624629 }630+ if metadata.codec_config {631+ flags |= FRAME_CHANNEL_FLAG_CODEC_CONFIG;632+ }625633 626634 let mut envelope = Vec::with_capacity(FRAME_CHANNEL_HEADER_LEN + payload.len());627635 envelope.extend_from_slice(&FRAME_CHANNEL_MAGIC);@@ -3400,6 +3408,7 @@ pub async fn iroh_client_connect(
34003408 capture_timestamp_micros,34013409 pts_micros,34023410 discontinuity,3411+ codec_config,34033412 ) = match media_transport {34043413 MediaTransport::UpstreamMoq => {34053414 let receiver = upstream_moq_media@@ -3454,6 +3463,7 @@ pub async fn iroh_client_connect(
34543463 Some(frame.header.capture_timestamp_us),34553464 Some(frame.header.pts_us),34563465 discontinuity,3466+ frame.header.flags.contains(FrameFlags::CODEC_CONFIG),34573467 );34583468 }34593469 }@@ -3534,6 +3544,7 @@ pub async fn iroh_client_connect(
35343544 Some(object.header.capture_timestamp_us),35353545 Some(object.header.pts_us),35363546 discontinuity,3547+ object.header.flags.contains(FrameFlags::CODEC_CONFIG),35373548 );35383549 }35393550 }@@ -3606,6 +3617,7 @@ pub async fn iroh_client_connect(
36063617 Some(frame.header.capture_timestamp_us),36073618 Some(frame.header.pts_us),36083619 discontinuity,3620+ frame.header.flags.contains(FrameFlags::CODEC_CONFIG),36093621 );36103622 }36113623 }@@ -3644,6 +3656,7 @@ pub async fn iroh_client_connect(
36443656 Some(frame.header.capture_timestamp_us),36453657 Some(frame.header.pts_us),36463658 frame.header.flags.contains(FrameFlags::DISCONTINUITY),3659+ frame.header.flags.contains(FrameFlags::CODEC_CONFIG),36473660 )36483661 }36493662 MediaTransport::LegacyV0 => {@@ -3693,7 +3706,18 @@ pub async fn iroh_client_connect(
36933706 }36943707 Ok(Ok(_)) => {}36953708 }3696- (w, h, frame_buf, is_keyframe, codec, None, None, None, false)3709+ (3710+ w,3711+ h,3712+ frame_buf,3713+ is_keyframe,3714+ codec,3715+ None,3716+ None,3717+ None,3718+ false,3719+ false,3720+ )36973721 }36983722 };36993723 lock_client_media_metrics(&metrics).observe_transport_receive(@@ -3748,6 +3772,7 @@ pub async fn iroh_client_connect(
37483772 codec: &codec,37493773 keyframe: is_keyframe,37503774 discontinuity,3775+ codec_config,37513776 sequence,37523777 capture_timestamp_micros,37533778 pts_micros,@@ -4468,6 +4493,7 @@ mod tests {
44684493 codec: "h264",44694494 keyframe: true,44704495 discontinuity: true,4496+ codec_config: true,44714497 sequence: Some(42),44724498 capture_timestamp_micros: Some(123_456),44734499 pts_micros: Some(98_765),@@ -4480,7 +4506,7 @@ mod tests {
44804506 assert_eq!(&envelope[0..4], b"SGFR");44814507 assert_eq!(envelope[4], 1);44824508 assert_eq!(envelope[5], 1);4483- assert_eq!(envelope[6], 0b11);4509+ assert_eq!(envelope[6], 0b111);44844510 assert_eq!(envelope[7], 0);44854511 assert_eq!(&envelope[8..10], &1280_u16.to_be_bytes());44864512 assert_eq!(&envelope[10..12], &800_u16.to_be_bytes());@@ -4500,6 +4526,7 @@ mod tests {
45004526 codec: "av1",45014527 keyframe: false,45024528 discontinuity: false,4529+ codec_config: false,45034530 sequence: None,45044531 capture_timestamp_micros: None,45054532 pts_micros: None,@@ -4521,12 +4548,23 @@ mod tests {
45214548 codec,45224549 keyframe: false,45234550 discontinuity: false,4551+ codec_config: false,45244552 sequence: None,45254553 capture_timestamp_micros: None,45264554 pts_micros: None,45274555 };45284556 assert!(encode_frame_envelope(metadata("vp9"), &[1]).is_err());45294557 assert!(encode_frame_envelope(metadata("h264"), &[]).is_err());4558+ assert!(4559+ encode_frame_envelope(4560+ FrameEnvelopeMetadata {4561+ codec_config: true,4562+ ..metadata("h264")4563+ },4564+ &[1]4565+ )4566+ .is_err()4567+ );45304568 assert!(45314569 encode_frame_envelope(45324570 FrameEnvelopeMetadata {