Client: H.264 decode → YUV→RGB8 → expand to RGBA → base64 → putImageData Was: H.264 decode → YUV→RGB8 → JPEG encode → base64 → Image decode → drawImage Removes JPEG encode + async Image load per frame. putImageData is synchronous and ~10x faster than createObjectURL+Image.onload cycle. Tradeoff: base64 RGBA is ~8MB/frame at 1080p vs ~30KB JPEG, but the Tauri event bus handles it and the CPU savings outweigh the bandwidth.
- commit
- 7459ecddbca652b7fe9c183160a28a527c9ffefc
- parent
- 0d3393013dde654d8735036577744aa6a9257a02
- author
- dami etoile <dami.etoile@gmail.com>
| M | src-tauri/ | 43 | ++++++++++++++---------------- |
| M | src/ | 15 | ++++------ |
@@ -957,8 +957,7 @@ pub async fn iroh_client_connect(
957957 break;958958 }959959 960- // Decode H.264 → YUV → RGB8 → JPEG961- // Split into individual NAL units — decoder needs them separately960+ // Decode H.264 → YUV → RGB8 → RGBA → base64 (no JPEG re-encode)962961 for nal in nal_units(&h264_buf) {963962 match decoder.decode(nal) {964963 Ok(Some(yuv)) => {@@ -967,26 +966,26 @@ pub async fn iroh_client_connect(
967966 let mut rgb_raw = vec![0u8; rgb_len];968967 yuv.write_rgb8(&mut rgb_raw);969968 970- // Re-encode as JPEG for canvas971- let img = image::RgbImage::from_raw(yw as u32, yh as u32, rgb_raw);972- if let Some(img) = img {973- let mut jpeg_buf = Vec::with_capacity(30_000);974- if image::DynamicImage::ImageRgb8(img)975- .write_to(&mut Cursor::new(&mut jpeg_buf), image::ImageFormat::Jpeg)976- .is_ok()977- {978- let b64 = base64::engine::general_purpose::STANDARD.encode(&jpeg_buf);979- let _ = app.emit(980- "frame",981- FramePayload {982- width: yw as u32,983- height: yh as u32,984- data: b64,985- },986- );987- }969+ // Expand RGB8 → RGBA for ImageData970+ let mut rgba_raw = vec![0u8; yw * yh * 4];971+ let npix = yw * yh;972+ for i in 0..npix {973+ rgba_raw[i * 4] = rgb_raw[i * 3];974+ rgba_raw[i * 4 + 1] = rgb_raw[i * 3 + 1];975+ rgba_raw[i * 4 + 2] = rgb_raw[i * 3 + 2];976+ rgba_raw[i * 4 + 3] = 255;988977 }989978 979+ let b64 = base64::engine::general_purpose::STANDARD.encode(&rgba_raw);980+ let _ = app.emit(981+ "frame",982+ FramePayload {983+ width: yw as u32,984+ height: yh as u32,985+ data: b64,986+ },987+ );988+990989 frame_count += 1;991990 let elapsed = start.elapsed();992991 let fps = frame_count as f64 / elapsed.as_secs_f64().max(0.001);@@ -995,9 +994,7 @@ pub async fn iroh_client_connect(
995994 serde_json::json!({ "fps": fps, "count": frame_count }),996995 );997996 }998- Ok(None) => {999- // Need more NALs — SPS/PPS or waiting for I-frame1000- }997+ Ok(None) => {}1001998 Err(e) => {1002999 eprintln!("[client] decode error: {}", e);10031000 }@@ -814,15 +814,12 @@
814814 frameHeight = height;815815 if (canvas.width !== width) canvas.width = width;816816 if (canvas.height !== height) canvas.height = height;817- const bytes = Uint8Array.from(atob(data), c => c.charCodeAt(0));818- const blob = new Blob([bytes], { type: 'image/jpeg' });819- const url = URL.createObjectURL(blob);820- const img = new Image();821- img.onload = () => {822- ctx.drawImage(img, 0, 0, canvas.width, canvas.height);823- URL.revokeObjectURL(url);824- };825- img.src = url;817+818+ // Decode base64 RGBA → Uint8Array → ImageData → putImageData819+ const rgba = Uint8Array.from(atob(data), c => c.charCodeAt(0));820+ const clamped = new Uint8ClampedArray(rgba.buffer);821+ const imageData = new ImageData(clamped, width, height);822+ ctx.putImageData(imageData, 0, 0);826823 });827824 828825 listen('frame-stats', (event) => {