ffmpeg handles both screen capture and H.264 encoding in one process: - Linux: x11grab input, auto-detects h264_nvenc/h264_vaapi/libx264 - macOS: avfoundation input, h264_videotoolbox - AUD insertion (h264_metadata BSF) for trivial frame boundary detection - NAL stream parsed for AUD boundaries to split into frames - Falls back to xcap+openh264 if ffmpeg not installed ffmpeg is a system dependency (apt install ffmpeg / brew install ffmpeg), not bundled. No binary bloat to Tauri. Expected improvement: 90ms capture + 80-150ms encode → ~5ms total with NVENC
- commit
- a0ec729a3f2dfadc32027cb4fd2675ef4c6e9414
- parent
- 2de22fd17607e3492493e69bd79a489d6c007b52
- author
- dami etoile <dami.etoile@gmail.com>
| M | src-tauri/ | 2 | +- |
| M | src-tauri/ | 288 | ++++++++++++++++++++++++++++++- |
| M | src/ | 6 | +- |
@@ -20,7 +20,7 @@ tauri-plugin-log = "2"
2020 ctap-hid-fido2 = "3.5"2121 iroh = "1.0"2222 anyhow = "1"23-tokio = { version = "1", features = ["macros", "rt-multi-thread", "io-util", "time"] }23+tokio = { version = "1", features = ["macros", "rt-multi-thread", "io-util", "time", "process"] }2424 xcap = "0.9"2525 image = { version = "0.25", default-features = false, features = ["jpeg"] }2626 base64 = "0.22"@@ -677,6 +677,293 @@ async fn stream_frames(conn: Connection, app: &AppHandle) -> anyhow::Result<()>
677677 return Ok(());678678 }679679 680+ // Detect screen dimensions (one-time xcap call)681+ let (w, h) = {682+ let monitors = xcap::Monitor::all().context("failed to enumerate monitors")?;683+ let mon = monitors684+ .into_iter()685+ .next()686+ .ok_or_else(|| anyhow::anyhow!("no monitors found"))?;687+ let img = mon.capture_image()?;688+ (img.width() as usize, img.height() as usize)689+ };690+ eprintln!("[host] screen resolution: {}x{}", w, h);691+692+ // Try ffmpeg subprocess first693+ if ffmpeg_available() {694+ eprintln!("[host] using ffmpeg subprocess for capture+encode");695+ match stream_frames_ffmpeg(&mut send, &mut recv, app, w, h).await {696+ Ok(()) => return Ok(()),697+ Err(e) => {698+ eprintln!("[host] ffmpeg failed: {}, falling back to xcap+openh264", e);699+ }700+ }701+ }702+703+ // Fallback: xcap + openh264704+ eprintln!("[host] using xcap + openh264");705+ stream_frames_xcap(&mut send, &mut recv, app).await706+}707+708+// ─── ffmpeg subprocess path ──────────────────────────────────────────────────709+710+fn ffmpeg_available() -> bool {711+ std::process::Command::new("ffmpeg")712+ .arg("-version")713+ .stdout(std::process::Stdio::null())714+ .stderr(std::process::Stdio::null())715+ .status()716+ .is_ok()717+}718+719+fn detect_ffmpeg_encoder() -> String {720+ // Check which hardware encoders are available721+ let encoders = std::process::Command::new("ffmpeg")722+ .arg("-encoders")723+ .output()724+ .ok()725+ .map(|o| String::from_utf8_lossy(&o.stdout).to_string())726+ .unwrap_or_default();727+728+ if cfg!(target_os = "macos") {729+ if encoders.contains("h264_videotoolbox") {730+ return "h264_videotoolbox".to_string();731+ }732+ } else {733+ if encoders.contains("h264_nvenc") {734+ return "h264_nvenc".to_string();735+ }736+ if encoders.contains("h264_vaapi") {737+ return "h264_vaapi".to_string();738+ }739+ }740+ "libx264".to_string()741+}742+743+/// Find the next AUD (NAL type 9) start code after `from`.744+fn find_next_aud(data: &[u8], from: usize) -> Option<usize> {745+ let mut i = from;746+ while i + 4 < data.len() {747+ if data[i] == 0 && data[i + 1] == 0 {748+ if data[i + 2] == 0 && i + 3 < data.len() && data[i + 3] == 1 {749+ if i + 4 < data.len() && (data[i + 4] & 0x1f) == 9 {750+ return Some(i);751+ }752+ i += 4;753+ continue;754+ }755+ if data[i + 2] == 1 {756+ if i + 3 < data.len() && (data[i + 3] & 0x1f) == 9 {757+ return Some(i);758+ }759+ i += 3;760+ continue;761+ }762+ }763+ i += 1;764+ }765+ None766+}767+768+/// Check if a frame's NAL data contains an IDR (type 5) or SPS (type 7).769+fn frame_is_keyframe(data: &[u8]) -> bool {770+ let mut i = 0;771+ while i + 3 < data.len() {772+ if data[i] == 0 && data[i + 1] == 0 {773+ let (sc_len, nal_off) = if data[i + 2] == 0 && i + 3 < data.len() && data[i + 3] == 1 {774+ (4, 4)775+ } else if data[i + 2] == 1 {776+ (3, 3)777+ } else {778+ i += 1;779+ continue;780+ };781+ if i + nal_off < data.len() {782+ let nal_type = data[i + nal_off] & 0x1f;783+ if nal_type == 5 || nal_type == 7 {784+ return true;785+ }786+ }787+ i += sc_len;788+ } else {789+ i += 1;790+ }791+ }792+ false793+}794+795+async fn stream_frames_ffmpeg(796+ send: &mut iroh::endpoint::SendStream,797+ recv: &mut iroh::endpoint::RecvStream,798+ app: &AppHandle,799+ width: usize,800+ height: usize,801+) -> anyhow::Result<()> {802+ use std::process::Stdio;803+ use tokio::io::AsyncReadExt;804+ use tokio::process::Command as TokioCommand;805+806+ let encoder = detect_ffmpeg_encoder();807+ eprintln!("[host] ffmpeg encoder: {}", encoder);808+809+ let mut cmd = TokioCommand::new("ffmpeg");810+ cmd.stdin(Stdio::null())811+ .stdout(Stdio::piped())812+ .stderr(Stdio::null())813+ .kill_on_drop(true);814+815+ // Input: screen capture816+ if cfg!(target_os = "macos") {817+ cmd.arg("-f").arg("avfoundation")818+ .arg("-framerate").arg("30")819+ .arg("-capture_cursor").arg("1")820+ .arg("-i").arg("1:");821+ } else {822+ let display = std::env::var("DISPLAY").unwrap_or_else(|_| ":0".to_string());823+ cmd.arg("-f").arg("x11grab")824+ .arg("-framerate").arg("30")825+ .arg("-video_size").arg(format!("{}x{}", width, height))826+ .arg("-i").arg(&display);827+ }828+829+ // Encoder-specific settings830+ match encoder.as_str() {831+ "h264_nvenc" => {832+ cmd.arg("-c:v").arg("h264_nvenc")833+ .arg("-preset").arg("p1")834+ .arg("-tune").arg("ll")835+ .arg("-rc").arg("cbr")836+ .arg("-b:v").arg("8M");837+ }838+ "h264_vaapi" => {839+ cmd.arg("-c:v").arg("h264_vaapi")840+ .arg("-rc_mode").arg("CBR")841+ .arg("-b:v").arg("8M");842+ }843+ "h264_videotoolbox" => {844+ cmd.arg("-c:v").arg("h264_videotoolbox")845+ .arg("-realtime").arg("1")846+ .arg("-b:v").arg("8M");847+ }848+ _ => {849+ // libx264 software fallback850+ cmd.arg("-c:v").arg("libx264")851+ .arg("-preset").arg("ultrafast")852+ .arg("-tune").arg("zerolatency")853+ .arg("-b:v").arg("8M");854+ }855+ }856+857+ // Common output settings858+ cmd.arg("-g").arg("30")859+ .arg("-bf").arg("0")860+ .arg("-pix_fmt").arg("yuv420p")861+ .arg("-bsf:v").arg("h264_metadata=aud=insert")862+ .arg("-f").arg("h264")863+ .arg("-");864+865+ let mut child = cmd.spawn().context("failed to spawn ffmpeg")?;866+ let mut stdout = child.stdout.take().context("no stdout from ffmpeg")?;867+868+ let mut buf: Vec<u8> = Vec::with_capacity(65536);869+ let mut frame_start: usize = 0;870+ let mut first_aud_seen = false;871+ let mut frame_count = 0u32;872+ let start = Instant::now();873+ let mut last_frame_time = Instant::now();874+875+ loop {876+ let mut tmp = [0u8; 16384];877+ let n = stdout.read(&mut tmp).await?;878+ if n == 0 {879+ eprintln!("[host] ffmpeg stdout closed");880+ break;881+ }882+ buf.extend_from_slice(&tmp[..n]);883+884+ // Extract complete frames (delimited by AUD NAL units)885+ loop {886+ let search_from = if first_aud_seen { frame_start + 6 } else { 0 };887+ match find_next_aud(&buf, search_from) {888+ Some(aud_pos) => {889+ if first_aud_seen {890+ // Frame data: buf[frame_start..aud_pos]891+ let frame_data = &buf[frame_start..aud_pos];892+ let is_keyframe = frame_is_keyframe(frame_data);893+ let frame_size = frame_data.len();894+895+ let now = Instant::now();896+ let frame_ms = now.duration_since(last_frame_time).as_secs_f64() * 1000.0;897+ last_frame_time = now;898+899+ // Header: width(4) + height(4) + size(4) + is_keyframe(1) = 13 bytes900+ let header = [901+ (width as u32).to_be_bytes(),902+ (height as u32).to_be_bytes(),903+ (frame_size as u32).to_be_bytes(),904+ ]905+ .concat();906+ let kf_byte = if is_keyframe { 1u8 } else { 0u8 };907+908+ send.write_all(&header).await?;909+ send.write_all(&[kf_byte]).await?;910+ send.write_all(frame_data).await?;911+912+ frame_count += 1;913+ let elapsed = start.elapsed();914+ let fps = frame_count as f64 / elapsed.as_secs_f64().max(0.001);915+916+ let _ = app.emit(917+ "host-encode-stats",918+ serde_json::json!({919+ "frame": frame_count,920+ "encode_ms": (frame_ms * 10.0).round() / 10.0,921+ "capture_ms": 0.0,922+ "size_bytes": frame_size,923+ "fps": (fps * 10.0).round() / 10.0,924+ "keyframe": is_keyframe,925+ }),926+ );927+928+ eprintln!(929+ "[host] frame={} {}x{} h264={}B kf={} fps={:.1} ftime={:.1}ms",930+ frame_count, width, height, frame_size, is_keyframe, fps, frame_ms931+ );932+ }933+ frame_start = aud_pos;934+ first_aud_seen = true;935+ }936+ None => break,937+ }938+ }939+940+ // Trim consumed data from buffer941+ if frame_start > 0 && frame_start >= buf.len() / 2 {942+ buf.drain(..frame_start);943+ frame_start = 0;944+ }945+946+ // Check for client disconnect947+ match tokio::time::timeout(Duration::from_millis(1), recv.read(&mut [0u8; 1])).await {948+ Ok(Ok(Some(_))) => {949+ eprintln!("[host] client disconnected");950+ break;951+ }952+ _ => {}953+ }954+ }955+956+ let _ = child.kill().await;957+ Ok(())958+}959+960+// ─── xcap + openh264 fallback path ───────────────────────────────────────────961+962+async fn stream_frames_xcap(963+ send: &mut iroh::endpoint::SendStream,964+ recv: &mut iroh::endpoint::RecvStream,965+ app: &AppHandle,966+) -> anyhow::Result<()> {680967 let monitors = xcap::Monitor::all().context("failed to enumerate monitors")?;681968 let monitor = monitors682969 .into_iter()@@ -749,7 +1036,6 @@ async fn stream_frames(conn: Connection, app: &AppHandle) -> anyhow::Result<()>
7491036 _ => {}7501037 }7511038 752- // Backpressure: yield to scheduler, no artificial throttle7531039 tokio::task::yield_now().await;7541040 }7551041 @@ -503,8 +503,8 @@
503503 </div>504504 <div class="panel-section">505505 <h3># performance</h3>506- <div class="info-row"><span class="info-label">encode</span><span class="info-value" id="perf-encode">— ms</span></div>507- <div class="info-row"><span class="info-label">capture</span><span class="info-value" id="perf-capture">— ms</span></div>506+ <div class="info-row"><span class="info-label">frame time</span><span class="info-value" id="perf-encode">— ms</span></div>507+ <div class="info-row"><span class="info-label">encoder</span><span class="info-value" id="perf-capture">—</span></div>508508 <div class="info-row"><span class="info-label">frame size</span><span class="info-value" id="perf-size">— KB</span></div>509509 <div class="info-row"><span class="info-label">fps</span><span class="info-value" id="perf-fps">—</span></div>510510 <canvas id="encode-spark" width="260" height="48" style="width:100%;height:48px;margin-top:8px;display:block;"></canvas>@@ -873,7 +873,7 @@
873873 listen('host-encode-stats', (event) => {874874 const { encode_ms, capture_ms, size_bytes, fps } = event.payload;875875 document.getElementById('perf-encode').textContent = encode_ms.toFixed(1) + ' ms';876- document.getElementById('perf-capture').textContent = capture_ms.toFixed(1) + ' ms';876+ document.getElementById('perf-capture').textContent = capture_ms > 0 ? capture_ms.toFixed(1) + ' ms' : 'ffmpeg';877877 document.getElementById('perf-size').textContent = (size_bytes / 1024).toFixed(1) + ' KB';878878 document.getElementById('perf-fps').textContent = fps.toFixed(1);879879