Host: xcap RGBA → RGB8 → YUVBuffer → openh264 Encoder → H.264 NAL bytes Client: H.264 NAL → openh264 Decoder → YUV → RGB8 → JPEG (for canvas) - P-frames are tiny vs full JPEGs — major bandwidth savings over Iroh relay - Encoder is Send, bitstream extracted to Vec before await (EncodedBitStream !Send) - Keyframe flag sent in header (13 bytes: w+h+size+kf) - Client re-encodes to JPEG for canvas (raw RGB8 base64 would be ~6MB/frame) - Installed nasm for 2-3x encode/decode speedup
- commit
- de5080291d77c631e6fea82ed5df1f94629c46a6
- parent
- e0af5fd3eed43df8c68c44f4f74b9dc5e9e7227f
- author
- dami etoile <dami.etoile@gmail.com>
| M | src-tauri/ | 62 | ++++++++++++++++ |
| M | src-tauri/ | 1 | + |
| M | src-tauri/ | 115 | +++++++++++++++++++++--------- |
@@ -144,6 +144,7 @@ dependencies = [
144144 "iroh",145145 "keyring",146146 "log",147+ "openh264",147148 "serde",148149 "serde_json",149150 "sha2 0.10.9",@@ -790,6 +791,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
790791 checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96"791792 dependencies = [792793 "find-msvc-tools",794+ "jobserver",795+ "libc",793796 "shlex 2.0.1",794797 ]795798 @@ -3290,6 +3293,16 @@ dependencies = [
32903293 "syn 2.0.118",32913294 ]32923295 3296+[[package]]3297+name = "jobserver"3298+version = "0.1.34"3299+source = "registry+https://github.com/rust-lang/crates.io-index"3300+checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"3301+dependencies = [3302+ "getrandom 0.3.4",3303+ "libc",3304+]3305+32933306 [[package]]32943307 name = "js-sys"32953308 version = "0.3.102"@@ -3761,6 +3774,15 @@ dependencies = [
37613774 "n0-future",37623775 ]37633776 3777+[[package]]3778+name = "nasm-rs"3779+version = "0.3.2"3780+source = "registry+https://github.com/rust-lang/crates.io-index"3781+checksum = "706bf8a5e8c8ddb99128c3291d31bd21f4bcde17f0f4c20ec678d85c74faa149"3782+dependencies = [3783+ "log",3784+]3785+37643786 [[package]]37653787 name = "ndk"37663788 version = "0.9.0"@@ -4552,6 +4574,27 @@ version = "0.3.1"
45524574 source = "registry+https://github.com/rust-lang/crates.io-index"45534575 checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"45544576 4577+[[package]]4578+name = "openh264"4579+version = "0.7.2"4580+source = "registry+https://github.com/rust-lang/crates.io-index"4581+checksum = "4e0b96c5808e7e5cf977056b4da08796895f993570b4a0ad658f90764c3c18fe"4582+dependencies = [4583+ "openh264-sys2",4584+ "wide",4585+]4586+4587+[[package]]4588+name = "openh264-sys2"4589+version = "0.7.1"4590+source = "registry+https://github.com/rust-lang/crates.io-index"4591+checksum = "881ad6de62993ff97a2cfbc63e5f127ecc930255718f9adc4c68c1dfeb308132"4592+dependencies = [4593+ "cc",4594+ "nasm-rs",4595+ "walkdir",4596+]4597+45554598 [[package]]45564599 name = "openssl-probe"45574600 version = "0.2.1"@@ -5533,6 +5576,15 @@ version = "1.0.23"
55335576 source = "registry+https://github.com/rust-lang/crates.io-index"55345577 checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"55355578 5579+[[package]]5580+name = "safe_arch"5581+version = "0.7.4"5582+source = "registry+https://github.com/rust-lang/crates.io-index"5583+checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323"5584+dependencies = [5585+ "bytemuck",5586+]5587+55365588 [[package]]55375589 name = "same-file"55385590 version = "1.0.6"@@ -7610,6 +7662,16 @@ dependencies = [
76107662 "windows-core 0.61.2",76117663 ]76127664 7665+[[package]]7666+name = "wide"7667+version = "0.7.33"7668+source = "registry+https://github.com/rust-lang/crates.io-index"7669+checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03"7670+dependencies = [7671+ "bytemuck",7672+ "safe_arch",7673+]7674+76137675 [[package]]76147676 name = "widestring"76157677 version = "1.2.1"@@ -25,6 +25,8 @@ use iroh::protocol::{ProtocolHandler, Router};
2525 use iroh::{Endpoint, SecretKey};2626 use serde::{Deserialize, Serialize};2727 use sha2::{Digest, Sha256};28+use openh264::encoder::Encoder;29+use openh264::formats::{RgbSliceU8, YUVBuffer, YUVSource};2830 use std::io::Cursor;2931 use std::sync::{Arc, Mutex};3032 use std::time::{Duration, Instant};@@ -613,34 +615,45 @@ async fn stream_frames(conn: Connection) -> anyhow::Result<()> {
613615 .next()614616 .ok_or_else(|| anyhow::anyhow!("no monitors found"))?;615617 618+ let mut encoder = Encoder::new().context("failed to create H.264 encoder")?;619+616620 let mut frame_count = 0u32;617621 let start = Instant::now();618622 619623 loop {620624 let image = monitor.capture_image()?;621625 let rgb_image = image::DynamicImage::ImageRgba8(image).to_rgb8();622- let (w, h) = (rgb_image.width(), rgb_image.height());623-624- let mut jpeg_buf = Vec::with_capacity(50_000);625- rgb_image.write_to(&mut Cursor::new(&mut jpeg_buf), image::ImageFormat::Jpeg)?;626- let jpeg_size = jpeg_buf.len();626+ let (w, h) = (rgb_image.width() as usize, rgb_image.height() as usize);627+628+ // RGB8 → YUV → H.264629+ let rgb_source = RgbSliceU8::new(rgb_image.as_raw(), (w, h));630+ let yuv = YUVBuffer::from_rgb8_source(rgb_source);631+ let (h264_data, is_keyframe) = {632+ let bitstream = encoder.encode(&yuv).context("H.264 encode failed")?;633+ let kf = matches!(bitstream.frame_type(), openh264::encoder::FrameType::I | openh264::encoder::FrameType::IDR);634+ (bitstream.to_vec(), kf)635+ };636+ let h264_size = h264_data.len();627637 638+ // Header: width(4) + height(4) + size(4) + is_keyframe(1)639+ let kf_byte = [0u8, 0u8, 0u8, if is_keyframe { 1u8 } else { 0u8 }];628640 let header = [629641 (w as u32).to_be_bytes(),630642 (h as u32).to_be_bytes(),631- (jpeg_size as u32).to_be_bytes(),643+ (h264_size as u32).to_be_bytes(),644+ kf_byte,632645 ]633646 .concat();634647 635648 send.write_all(&header).await?;636- send.write_all(&jpeg_buf).await?;649+ send.write_all(&h264_data).await?;637650 638651 frame_count += 1;639652 let elapsed = start.elapsed();640653 let fps = frame_count as f64 / elapsed.as_secs_f64().max(0.001);641654 eprintln!(642- "[host] frame={} {}x{} jpeg={}B fps={:.1}",643- frame_count, w, h, jpeg_size, fps655+ "[host] frame={} {}x{} h264={}B kf={} fps={:.1}",656+ frame_count, w, h, h264_size, is_keyframe, fps644657 );645658 646659 match tokio::time::timeout(Duration::from_millis(1), recv.read(&mut [0u8; 1])).await {@@ -846,46 +859,82 @@ pub async fn iroh_client_connect(
846859 let _ = input_stream.finish();847860 });848861 849- // Spawn frame reader862+ // Spawn frame reader (H.264 decode → JPEG for canvas)850863 tokio::spawn(async move {864+ let mut decoder = match openh264::decoder::Decoder::new() {865+ Ok(d) => d,866+ Err(e) => {867+ let _ = app.emit("frame-error", format!("Decoder init failed: {}", e));868+ return;869+ }870+ };871+851872 let mut frame_count = 0u32;852873 let start = Instant::now();853874 854875 loop {855- let mut header = [0u8; 12];876+ // Header: width(4) + height(4) + size(4) + is_keyframe(1) = 13 bytes877+ let mut header = [0u8; 13];856878 if frame_recv.read_exact(&mut header).await.is_err() {857879 let _ = app.emit("frame-error", "Connection lost");858880 break;859881 }860882 861- let w = u32::from_be_bytes([header[0], header[1], header[2], header[3]]);862- let h = u32::from_be_bytes([header[4], header[5], header[6], header[7]]);863- let jpeg_len =883+ let _w = u32::from_be_bytes([header[0], header[1], header[2], header[3]]);884+ let _h = u32::from_be_bytes([header[4], header[5], header[6], header[7]]);885+ let h264_len =864886 u32::from_be_bytes([header[8], header[9], header[10], header[11]]) as usize;887+ let _is_keyframe = header[12] == 1;865888 866- let mut jpeg_buf = vec![0u8; jpeg_len];867- if frame_recv.read_exact(&mut jpeg_buf).await.is_err() {889+ let mut h264_buf = vec![0u8; h264_len];890+ if frame_recv.read_exact(&mut h264_buf).await.is_err() {868891 let _ = app.emit("frame-error", "Connection lost");869892 break;870893 }871894 872- frame_count += 1;873- let elapsed = start.elapsed();874- let fps = frame_count as f64 / elapsed.as_secs_f64().max(0.001);875-876- let b64 = base64::engine::general_purpose::STANDARD.encode(&jpeg_buf);877- let _ = app.emit(878- "frame",879- FramePayload {880- width: w,881- height: h,882- data: b64,883- },884- );885- let _ = app.emit(886- "frame-stats",887- serde_json::json!({ "fps": fps, "count": frame_count }),888- );895+ // Decode H.264 → YUV → RGB8 → JPEG896+ match decoder.decode(&h264_buf) {897+ Ok(Some(yuv)) => {898+ let (yw, yh) = yuv.dimensions();899+ let rgb_len = yuv.rgb8_len();900+ let mut rgb_raw = vec![0u8; rgb_len];901+ yuv.write_rgb8(&mut rgb_raw);902+903+ // Re-encode as JPEG for canvas904+ let img = image::RgbImage::from_raw(yw as u32, yh as u32, rgb_raw);905+ if let Some(img) = img {906+ let mut jpeg_buf = Vec::with_capacity(30_000);907+ if image::DynamicImage::ImageRgb8(img)908+ .write_to(&mut Cursor::new(&mut jpeg_buf), image::ImageFormat::Jpeg)909+ .is_ok()910+ {911+ let b64 = base64::engine::general_purpose::STANDARD.encode(&jpeg_buf);912+ let _ = app.emit(913+ "frame",914+ FramePayload {915+ width: yw as u32,916+ height: yh as u32,917+ data: b64,918+ },919+ );920+ }921+ }922+923+ frame_count += 1;924+ let elapsed = start.elapsed();925+ let fps = frame_count as f64 / elapsed.as_secs_f64().max(0.001);926+ let _ = app.emit(927+ "frame-stats",928+ serde_json::json!({ "fps": fps, "count": frame_count }),929+ );930+ }931+ Ok(None) => {932+ // Need more data — normal for first few NALs933+ }934+ Err(e) => {935+ eprintln!("[client] decode error: {}", e);936+ }937+ }889938 }890939 891940 drop(endpoint);