WebKit/WebKitGTK (Tauri's webview) doesn't implement the WebCodecs API. VideoDecoder/EncodedVideoChunk don't exist, so frames silently failed and control broke. Reverted to working pipeline: H.264 → openh264 decode → YUV→RGB8 → JPEG → base64 → Image → canvas This is the same pipeline that was working at 1.2-1.6fps before. The real bottleneck is host-side encode time, not client decode.
- commit
- b80b4c1c130d38353215420ea838b9b30cf44c0d
- parent
- a3aee6b154a3bd8276251c5d9385c140541a9b94
- author
- dami etoile <dami.etoile@gmail.com>
| M | src-tauri/ | 67 | ++++++++++++++++++++--------- |
| M | src/ | 69 | ++++-------------------------- |
@@ -924,13 +924,20 @@ pub async fn iroh_client_connect(
924924 let _ = input_stream.finish();925925 });926926 927- // Spawn frame reader (H.264 pass-through to frontend WebCodecs decoder)927+ // Spawn frame reader (H.264 decode → JPEG for canvas)928928 tokio::spawn(async move {929+ let mut decoder = match openh264::decoder::Decoder::new() {930+ Ok(d) => d,931+ Err(e) => {932+ let _ = app.emit("frame-error", format!("Decoder init failed: {}", e));933+ return;934+ }935+ };936+929937 let mut frame_count = 0u32;930938 let start = Instant::now();931939 932940 loop {933- // Header: width(4) + height(4) + size(4) + is_keyframe(1) = 13 bytes934941 let mut header = [0u8; 13];935942 if frame_recv.read_exact(&mut header).await.is_err() {936943 let _ = app.emit("frame-error", "Connection lost");@@ -949,24 +956,44 @@ pub async fn iroh_client_connect(
949956 break;950957 }951958 952- // Pass H.264 bytes directly to frontend — browser decodes via WebCodecs953- let b64 = base64::engine::general_purpose::STANDARD.encode(&h264_buf);954- let _ = app.emit(955- "frame",956- FramePayload {957- width: w,958- height: h,959- data: b64,960- },961- );962-963- frame_count += 1;964- let elapsed = start.elapsed();965- let fps = frame_count as f64 / elapsed.as_secs_f64().max(0.001);966- let _ = app.emit(967- "frame-stats",968- serde_json::json!({ "fps": fps, "count": frame_count, "keyframe": is_keyframe }),969- );959+ for nal in nal_units(&h264_buf) {960+ if let Ok(Some(yuv)) = decoder.decode(nal) {961+ let (yw, yh) = yuv.dimensions();962+ let rgb_len = yuv.rgb8_len();963+ let mut rgb_raw = vec![0u8; rgb_len];964+ yuv.write_rgb8(&mut rgb_raw);965+966+ let img = match image::RgbImage::from_raw(yw as u32, yh as u32, rgb_raw) {967+ Some(img) => img,968+ None => continue,969+ };970+ let mut jpeg_buf = Vec::with_capacity(30_000);971+ if image::DynamicImage::ImageRgb8(img)972+ .write_to(&mut Cursor::new(&mut jpeg_buf), image::ImageFormat::Jpeg)973+ .is_err()974+ {975+ continue;976+ }977+978+ let b64 = base64::engine::general_purpose::STANDARD.encode(&jpeg_buf);979+ let _ = app.emit(980+ "frame",981+ FramePayload {982+ width: yw as u32,983+ height: yh as u32,984+ data: b64,985+ },986+ );987+988+ frame_count += 1;989+ let elapsed = start.elapsed();990+ let fps = frame_count as f64 / elapsed.as_secs_f64().max(0.001);991+ let _ = app.emit(992+ "frame-stats",993+ serde_json::json!({ "fps": fps, "count": frame_count, "keyframe": is_keyframe }),994+ );995+ }996+ }970997 }971998 972999 drop(endpoint);@@ -750,13 +750,6 @@
750750 // Clear intro status fields751751 document.getElementById('intro-status-unreg').textContent = '';752752 document.getElementById('intro-status-reg').textContent = '';753- // Reset WebCodecs decoder754- if (videoDecoder) {755- try { videoDecoder.close(); } catch(e) {}756- videoDecoder = null;757- decoderConfigured = false;758- pendingKeyframe = true;759- }760753 }761754 762755 // ─── Unregister ─────────────────────────────────────────────────────@@ -811,67 +804,25 @@
811804 document.getElementById('host-conn-count').textContent = event.payload;812805 });813806 814- // ─── Frames — WebCodecs hardware H.264 decode ───────────────────────807+ // ─── Frames ─────────────────────────────────────────────────────────815808 const canvas = document.getElementById('frame-canvas');816809 const ctx = canvas.getContext('2d');817810 818- let videoDecoder = null;819- let decoderConfigured = false;820- let frameQueue = [];821- let pendingKeyframe = true;822-823- function initDecoder(width, height) {824- if (videoDecoder) {825- try { videoDecoder.close(); } catch(e) {}826- }827- videoDecoder = new VideoDecoder({828- output: (frame) => {829- ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);830- frame.close();831- },832- error: (e) => {833- console.error('VideoDecoder error:', e);834- pendingKeyframe = true;835- }836- });837- videoDecoder.configure({838- codec: 'avc1.42001e',839- codedWidth: width,840- codedHeight: height,841- });842- decoderConfigured = true;843- pendingKeyframe = true;844- }845-846811 listen('frame', (event) => {847812 const { width, height, data } = event.payload;848813 frameWidth = width;849814 frameHeight = height;850815 if (canvas.width !== width) canvas.width = width;851816 if (canvas.height !== height) canvas.height = height;852-853- // Decode base64 → Uint8Array854- const h264 = Uint8Array.from(atob(data), c => c.charCodeAt(0));855-856- // Configure decoder on first frame857- if (!decoderConfigured) {858- initDecoder(width, height);859- }860-861- // Create EncodedVideoChunk — let the browser split NALs internally862- const chunk = new EncodedVideoChunk({863- type: pendingKeyframe ? 'key' : 'delta',864- timestamp: performance.now() * 1000,865- data: h264,866- });867-868- try {869- videoDecoder.decode(chunk);870- pendingKeyframe = false;871- } catch (e) {872- console.warn('decode failed, waiting for keyframe:', e);873- pendingKeyframe = true;874- }817+ const bytes = Uint8Array.from(atob(data), c => c.charCodeAt(0));818+ const blob = new Blob([bytes], { type: 'image/jpeg' });819+ const url = URL.createObjectURL(blob);820+ const img = new Image();821+ img.onload = () => {822+ ctx.drawImage(img, 0, 0, canvas.width, canvas.height);823+ URL.revokeObjectURL(url);824+ };825+ img.src = url;875826 });876827 877828 listen('frame-stats', (event) => {