Replaces the entire client-side software decode pipeline: Was: H.264 → openh264 decode → YUV→RGB8 → JPEG encode → base64 → Image decode → canvas Now: H.264 → base64 → EncodedVideoChunk → VideoDecoder → VideoFrame → canvas Browser handles H.264 decode in hardware. No openh264 on client, no JPEG, no async Image loading. Rust just passes H.264 bytes through. JetKVM-inspired: they use WebRTC <video> for hardware decode. We use WebCodecs VideoDecoder (same hardware path, but works over Iroh not WebRTC). - Client Rust: pure pass-through, no decode, no image crate - Frontend: VideoDecoder.configure(avc1.42001e) → decode each chunk - Disconnect resets decoder state - Stats show [kf] marker on keyframes
- commit
- a3aee6b154a3bd8276251c5d9385c140541a9b94
- parent
- 7459ecddbca652b7fe9c183160a28a527c9ffefc
- author
- dami etoile <dami.etoile@gmail.com>
| M | src-tauri/ | 77 | +++++++++--------------------- |
| M | src/ | 67 | +++++++++++++++++++++++--- |
@@ -924,16 +924,8 @@ pub async fn iroh_client_connect(
924924 let _ = input_stream.finish();925925 });926926 927- // Spawn frame reader (H.264 decode → JPEG for canvas)927+ // Spawn frame reader (H.264 pass-through to frontend WebCodecs decoder)928928 tokio::spawn(async move {929- let mut decoder = match openh264::decoder::Decoder::new() {930- Ok(d) => d,931- Err(e) => {932- let _ = app.emit("frame-error", format!("Decoder init failed: {}", e));933- return;934- }935- };936-937929 let mut frame_count = 0u32;938930 let start = Instant::now();939931 @@ -945,11 +937,11 @@ pub async fn iroh_client_connect(
945937 break;946938 }947939 948- let _w = u32::from_be_bytes([header[0], header[1], header[2], header[3]]);949- let _h = u32::from_be_bytes([header[4], header[5], header[6], header[7]]);940+ let w = u32::from_be_bytes([header[0], header[1], header[2], header[3]]);941+ let h = u32::from_be_bytes([header[4], header[5], header[6], header[7]]);950942 let h264_len =951943 u32::from_be_bytes([header[8], header[9], header[10], header[11]]) as usize;952- let _is_keyframe = header[12] == 1;944+ let is_keyframe = header[12] == 1;953945 954946 let mut h264_buf = vec![0u8; h264_len];955947 if frame_recv.read_exact(&mut h264_buf).await.is_err() {@@ -957,49 +949,24 @@ pub async fn iroh_client_connect(
957949 break;958950 }959951 960- // Decode H.264 → YUV → RGB8 → RGBA → base64 (no JPEG re-encode)961- for nal in nal_units(&h264_buf) {962- match decoder.decode(nal) {963- Ok(Some(yuv)) => {964- let (yw, yh) = yuv.dimensions();965- let rgb_len = yuv.rgb8_len();966- let mut rgb_raw = vec![0u8; rgb_len];967- yuv.write_rgb8(&mut rgb_raw);968-969- // Expand RGB8 → RGBA for ImageData970- let mut rgba_raw = vec![0u8; yw * yh * 4];971- let npix = yw * yh;972- for i in 0..npix {973- rgba_raw[i * 4] = rgb_raw[i * 3];974- rgba_raw[i * 4 + 1] = rgb_raw[i * 3 + 1];975- rgba_raw[i * 4 + 2] = rgb_raw[i * 3 + 2];976- rgba_raw[i * 4 + 3] = 255;977- }978-979- let b64 = base64::engine::general_purpose::STANDARD.encode(&rgba_raw);980- let _ = app.emit(981- "frame",982- FramePayload {983- width: yw as u32,984- height: yh as u32,985- data: b64,986- },987- );988-989- frame_count += 1;990- let elapsed = start.elapsed();991- let fps = frame_count as f64 / elapsed.as_secs_f64().max(0.001);992- let _ = app.emit(993- "frame-stats",994- serde_json::json!({ "fps": fps, "count": frame_count }),995- );996- }997- Ok(None) => {}998- Err(e) => {999- eprintln!("[client] decode error: {}", e);1000- }1001- }1002- }952+ // Pass H.264 bytes directly to frontend — browser decodes via WebCodecs953+ let b64 = base64::engine::general_purpose::STANDARD.encode(&h264_buf);954+ let _ = app.emit(955+ "frame",956+ FramePayload {957+ width: w,958+ height: h,959+ data: b64,960+ },961+ );962+963+ frame_count += 1;964+ let elapsed = start.elapsed();965+ let fps = frame_count as f64 / elapsed.as_secs_f64().max(0.001);966+ let _ = app.emit(967+ "frame-stats",968+ serde_json::json!({ "fps": fps, "count": frame_count, "keyframe": is_keyframe }),969+ );1003970 }1004971 1005972 drop(endpoint);@@ -750,6 +750,13 @@
750750 // Clear intro status fields751751 document.getElementById('intro-status-unreg').textContent = '';752752 document.getElementById('intro-status-reg').textContent = '';753+ // Reset WebCodecs decoder754+ if (videoDecoder) {755+ try { videoDecoder.close(); } catch(e) {}756+ videoDecoder = null;757+ decoderConfigured = false;758+ pendingKeyframe = true;759+ }753760 }754761 755762 // ─── Unregister ─────────────────────────────────────────────────────@@ -804,10 +811,38 @@
804811 document.getElementById('host-conn-count').textContent = event.payload;805812 });806813 807- // ─── Frames ─────────────────────────────────────────────────────────814+ // ─── Frames — WebCodecs hardware H.264 decode ───────────────────────808815 const canvas = document.getElementById('frame-canvas');809816 const ctx = canvas.getContext('2d');810817 818+ let videoDecoder = null;819+ let decoderConfigured = false;820+ let frameQueue = [];821+ let pendingKeyframe = true;822+823+ function initDecoder(width, height) {824+ if (videoDecoder) {825+ try { videoDecoder.close(); } catch(e) {}826+ }827+ videoDecoder = new VideoDecoder({828+ output: (frame) => {829+ ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);830+ frame.close();831+ },832+ error: (e) => {833+ console.error('VideoDecoder error:', e);834+ pendingKeyframe = true;835+ }836+ });837+ videoDecoder.configure({838+ codec: 'avc1.42001e',839+ codedWidth: width,840+ codedHeight: height,841+ });842+ decoderConfigured = true;843+ pendingKeyframe = true;844+ }845+811846 listen('frame', (event) => {812847 const { width, height, data } = event.payload;813848 frameWidth = width;@@ -815,16 +850,34 @@
815850 if (canvas.width !== width) canvas.width = width;816851 if (canvas.height !== height) canvas.height = height;817852 818- // Decode base64 RGBA → Uint8Array → ImageData → putImageData819- const rgba = Uint8Array.from(atob(data), c => c.charCodeAt(0));820- const clamped = new Uint8ClampedArray(rgba.buffer);821- const imageData = new ImageData(clamped, width, height);822- ctx.putImageData(imageData, 0, 0);853+ // Decode base64 → Uint8Array854+ const h264 = Uint8Array.from(atob(data), c => c.charCodeAt(0));855+856+ // Configure decoder on first frame857+ if (!decoderConfigured) {858+ initDecoder(width, height);859+ }860+861+ // Create EncodedVideoChunk — let the browser split NALs internally862+ const chunk = new EncodedVideoChunk({863+ type: pendingKeyframe ? 'key' : 'delta',864+ timestamp: performance.now() * 1000,865+ data: h264,866+ });867+868+ try {869+ videoDecoder.decode(chunk);870+ pendingKeyframe = false;871+ } catch (e) {872+ console.warn('decode failed, waiting for keyframe:', e);873+ pendingKeyframe = true;874+ }823875 });824876 825877 listen('frame-stats', (event) => {826878 const stats = document.getElementById('stats');827- stats.textContent = `${event.payload.fps.toFixed(1)} fps · ${event.payload.count}`;879+ const kf = event.payload.keyframe ? ' [kf]' : '';880+ stats.textContent = `${event.payload.fps.toFixed(1)} fps · ${event.payload.count}${kf}`;828881 stats.classList.add('visible');829882 });830883