fix: WebCodecs decoder failure — wrong codec level + missing keyframe flag

dami etoile committed Jun 23944dccf

Root causes:
1. Hardcoded 'avc1.42001e' = Baseline Level 3.0 (max 720x576).
   1920x1080 needs Level 4.0+. Decoder rejects → EncodingError.
2. Keyframe detection was guessing (pendingKeyframe var), not using
   the actual keyframe flag from the backend.

Fixes:
- Parse SPS NAL from first H.264 packet → extract real profile/constraints/level
  → build correct codec string (e.g. 'avc1.42e028' for Baseline L4.0)
- Fallback codec: 'avc1.42e028' (Baseline L4.0, supports 1080p)
- Backend sends keyframe flag in FramePayload
- Frontend uses payload.keyframe for EncodedVideoChunk type
- Added optimizeForRealtimeUse: true
- Console logs codec string + dimensions on init

commit: 944dccf22df891b58c74988c7847e175c08946d6
parent: f3bb53ff444cee450cec22957400913887f1d842
author: dami etoile <dami.etoile@gmail.com>

2 files changed, +37 -11wrap lines

M	src-tauri/src/commands.rs	5	++-
M	src/ui/index.html	43	++++++++++++++++++++++++------

M src-tauri/src/commands.rs +3 -2

@@ -835,6 +835,7 @@ pub struct FramePayload {
835835     pub width: u32,836836     pub height: u32,837837     pub data: String,838+    pub keyframe: bool,838839 }839840 840841 #[derive(Serialize)]@@ -980,7 +981,7 @@ pub async fn iroh_client_connect(
980981                 let b64 = base64::engine::general_purpose::STANDARD.encode(&h264_buf);981982                 let _ = app.emit(982983                     "frame",983-                    FramePayload { width: w, height: h, data: b64 },984+                    FramePayload { width: w, height: h, data: b64, keyframe: is_keyframe },984985                 );985986 986987                 frame_count += 1;@@ -1014,7 +1015,7 @@ pub async fn iroh_client_connect(
10141015                         let b64 = base64::engine::general_purpose::STANDARD.encode(&jpeg_buf);10151016                         let _ = app.emit(10161017                             "frame",1017-                            FramePayload { width: yw as u32, height: yh as u32, data: b64 },1018+                            FramePayload { width: yw as u32, height: yh as u32, data: b64, keyframe: is_keyframe },10181019                         );10191020 10201021                         frame_count += 1;

M src/ui/index.html +34 -9

@@ -813,7 +813,6 @@
813813     let hasWebCodecs = ('VideoDecoder' in window);814814     let videoDecoder = null;815815     let decoderConfigured = false;816-    let pendingKeyframe = true;817816 818817     (async function detectWebCodecs() {819818       const el = document.getElementById('codec-webcodecs');@@ -831,7 +830,31 @@
831830       }832831     })();833832 834-    function initWebCodecsDecoder(width, height) {833+    function parseSpsCodec(h264Data) {834+      // Scan for SPS NAL (type 7) and extract profile/level835+      let i = 0;836+      while (i < h264Data.length - 4) {837+        if (h264Data[i] === 0 && h264Data[i+1] === 0 && h264Data[i+2] === 0 && h264Data[i+3] === 1) {838+          const nalType = h264Data[i+4] & 0x1f;839+          if (nalType === 7) { // SPS840+            const profileIdc = h264Data[i+5];841+            const constraintFlags = h264Data[i+6];842+            const levelIdc = h264Data[i+7];843+            const profileHex = profileIdc.toString(16).padStart(2, '0');844+            const constraintHex = constraintFlags.toString(16).padStart(2, '0');845+            const levelHex = levelIdc.toString(16).padStart(2, '0');846+            return `avc1.${profileHex}${constraintHex}${levelHex}`;847+          }848+          i += 4;849+        } else {850+          i++;851+        }852+      }853+      // Fallback: Baseline, Level 4.0 (supports 1080p)854+      return 'avc1.42e028';855+    }856+857+    function initWebCodecsDecoder(width, height, codecStr) {835858       if (videoDecoder) {836859         try { videoDecoder.close(); } catch(e) {}837860       }@@ -842,16 +865,15 @@
842865         },843866         error: (e) => {844867           console.error('VideoDecoder error:', e);845-          pendingKeyframe = true;846868         }847869       });848870       videoDecoder.configure({849-        codec: 'avc1.42001e',871+        codec: codecStr,850872         codedWidth: width,851873         codedHeight: height,874+        optimizeForRealtimeUse: true,852875       });853876       decoderConfigured = true;854-      pendingKeyframe = true;855877     }856878 857879     // ─── Frames — dual path: WebCodecs or openh264→JPEG ─────────────────@@ -868,17 +890,20 @@
868890       if (hasWebCodecs) {869891         // Path 1: WebCodecs hardware decode870892         const h264 = Uint8Array.from(atob(data), c => c.charCodeAt(0));871-        if (!decoderConfigured) initWebCodecsDecoder(width, height);893+        if (!decoderConfigured) {894+          const codecStr = parseSpsCodec(h264);895+          console.log('WebCodecs codec:', codecStr, 'w:', width, 'h:', height);896+          initWebCodecsDecoder(width, height, codecStr);897+        }872898         const chunk = new EncodedVideoChunk({873-          type: pendingKeyframe ? 'key' : 'delta',899+          type: event.payload.keyframe ? 'key' : 'delta',874900           timestamp: performance.now() * 1000,875901           data: h264,876902         });877903         try {878904           videoDecoder.decode(chunk);879-          pendingKeyframe = false;880905         } catch (e) {881-          pendingKeyframe = true;906+          console.warn('decode failed:', e);882907         }883908       } else {884909         // Path 2: JPEG from backend (openh264 decode → JPEG encode in Rust)