When frontend reports WebCodecs available: Backend sends raw H.264 NAL bytes → browser decodes in hardware No openh264 decode, no JPEG re-encode, no image crate on client When WebCodecs unavailable: Falls back to openh264 decode → JPEG → canvas (existing path) WebKitGTK 2.44+ supports WebCodecs via GStreamer. Server has 2.52.3. Installed gstreamer1.0-libav + plugins-bad + vaapi for H.264 decode. macOS WKWebView has WebCodecs since Safari 17. Info pane shows which path is active.
- commit
- f3bb53ff444cee450cec22957400913887f1d842
- parent
- 0ec1bb5543e046a9c68a075228794fc5f946782a
- author
- dami etoile <dami.etoile@gmail.com>
| M | src-tauri/ | 110 | ++++++++++++++++++------------ |
@@ -937,19 +937,25 @@ pub async fn iroh_client_connect(
937937 let _ = input_stream.finish();938938 });939939 940- // Spawn frame reader (H.264 decode → JPEG for canvas)940+ // Spawn frame reader — dual path based on WebCodecs availability941+ let use_webcodecs = state.webcodecs.load(Ordering::SeqCst);941942 tokio::spawn(async move {942- let mut decoder = match openh264::decoder::Decoder::new() {943- Ok(d) => d,944- Err(e) => {945- let _ = app.emit("frame-error", format!("Decoder init failed: {}", e));946- return;947- }948- };949-950943 let mut frame_count = 0u32;951944 let start = Instant::now();952945 946+ // Initialize software decoder only if WebCodecs is NOT available947+ let mut decoder = if use_webcodecs {948+ None949+ } else {950+ match openh264::decoder::Decoder::new() {951+ Ok(d) => Some(d),952+ Err(e) => {953+ let _ = app.emit("frame-error", format!("Decoder init failed: {}", e));954+ return;955+ }956+ }957+ };958+953959 loop {954960 let mut header = [0u8; 13];955961 if frame_recv.read_exact(&mut header).await.is_err() {@@ -969,44 +975,60 @@ pub async fn iroh_client_connect(
969975 break;970976 }971977 972- for nal in nal_units(&h264_buf) {973- if let Ok(Some(yuv)) = decoder.decode(nal) {974- let (yw, yh) = yuv.dimensions();975- let rgb_len = yuv.rgb8_len();976- let mut rgb_raw = vec![0u8; rgb_len];977- yuv.write_rgb8(&mut rgb_raw);978-979- let img = match image::RgbImage::from_raw(yw as u32, yh as u32, rgb_raw) {980- Some(img) => img,981- None => continue,982- };983- let mut jpeg_buf = Vec::with_capacity(30_000);984- if image::DynamicImage::ImageRgb8(img)985- .write_to(&mut Cursor::new(&mut jpeg_buf), image::ImageFormat::Jpeg)986- .is_err()987- {988- continue;978+ if use_webcodecs {979+ // Path 1: Pass raw H.264 to frontend — browser decodes via WebCodecs980+ let b64 = base64::engine::general_purpose::STANDARD.encode(&h264_buf);981+ let _ = app.emit(982+ "frame",983+ FramePayload { width: w, height: h, data: b64 },984+ );985+986+ frame_count += 1;987+ let elapsed = start.elapsed();988+ let fps = frame_count as f64 / elapsed.as_secs_f64().max(0.001);989+ let _ = app.emit(990+ "frame-stats",991+ serde_json::json!({ "fps": fps, "count": frame_count, "keyframe": is_keyframe }),992+ );993+ } else if let Some(ref mut dec) = decoder {994+ // Path 2: Software decode → JPEG995+ for nal in nal_units(&h264_buf) {996+ if let Ok(Some(yuv)) = dec.decode(nal) {997+ let (yw, yh) = yuv.dimensions();998+ let rgb_len = yuv.rgb8_len();999+ let mut rgb_raw = vec![0u8; rgb_len];1000+ yuv.write_rgb8(&mut rgb_raw);1001+1002+ let img = match image::RgbImage::from_raw(yw as u32, yh as u32, rgb_raw) {1003+ Some(img) => img,1004+ None => continue,1005+ };1006+ let mut jpeg_buf = Vec::with_capacity(30_000);1007+ if image::DynamicImage::ImageRgb8(img)1008+ .write_to(&mut Cursor::new(&mut jpeg_buf), image::ImageFormat::Jpeg)1009+ .is_err()1010+ {1011+ continue;1012+ }1013+1014+ let b64 = base64::engine::general_purpose::STANDARD.encode(&jpeg_buf);1015+ let _ = app.emit(1016+ "frame",1017+ FramePayload { width: yw as u32, height: yh as u32, data: b64 },1018+ );1019+1020+ frame_count += 1;1021+ let elapsed = start.elapsed();1022+ let fps = frame_count as f64 / elapsed.as_secs_f64().max(0.001);1023+ let _ = app.emit(1024+ "frame-stats",1025+ serde_json::json!({ "fps": fps, "count": frame_count, "keyframe": is_keyframe }),1026+ );9891027 }990-991- let b64 = base64::engine::general_purpose::STANDARD.encode(&jpeg_buf);992- let _ = app.emit(993- "frame",994- FramePayload {995- width: yw as u32,996- height: yh as u32,997- data: b64,998- },999- );1000-1001- frame_count += 1;1002- let elapsed = start.elapsed();1003- let fps = frame_count as f64 / elapsed.as_secs_f64().max(0.001);1004- let _ = app.emit(1005- "frame-stats",1006- serde_json::json!({ "fps": fps, "count": frame_count, "keyframe": is_keyframe }),1007- );10081028 }10091029 }1030+1031+ tokio::task::yield_now().await;10101032 }10111033 10121034 drop(endpoint);