- Runtime detection: 'VideoDecoder' in window - Path 1 (WebCodecs): H.264 → EncodedVideoChunk → VideoDecoder → canvas - Path 2 (fallback): H.264 → openh264 decode → JPEG → Image → canvas - Backend: set_webcodecs_available / is_webcodecs_available commands - Info pane shows: webcodecs status + active decode path - Researched ffmpeg.wasm — not suitable for streaming (file-based API, 20MB binary, SharedArrayBuffer requirements, no real-time NAL decode)
- commit
- 0ec1bb5543e046a9c68a075228794fc5f946782a
- parent
- e20bc07fab8484c4e5fa6be11b50e19255f81fb9
- author
- dami etoile <dami.etoile@gmail.com>
| M | src-tauri/ | 12 | ++++ |
| M | src-tauri/ | 2 | + |
| M | src/ | 89 | +++++++++++++++++++++++++++--- |
@@ -305,6 +305,7 @@ pub struct AppState {
305305 pub host_endpoint: TokioMutex<Option<Endpoint>>,306306 pub input_send: TokioMutex<Option<tokio::sync::mpsc::UnboundedSender<InputEvent>>>,307307 pub client_endpoint: TokioMutex<Option<Endpoint>>,308+ pub webcodecs: std::sync::atomic::AtomicBool,308309 pub daemon: bool,309310 }310311 @@ -321,6 +322,7 @@ impl Default for AppState {
321322 host_endpoint: TokioMutex::new(None),322323 input_send: TokioMutex::new(None),323324 client_endpoint: TokioMutex::new(None),325+ webcodecs: std::sync::atomic::AtomicBool::new(false),324326 daemon,325327 }326328 }@@ -473,6 +475,16 @@ pub fn is_daemon_mode(state: State<'_, AppState>) -> bool {
473475 state.daemon474476 }475477 478+479+pub fn set_webcodecs_available(state: State<'_, AppState>, available: bool) {480+ state.webcodecs.store(available, Ordering::SeqCst);481+}482+483+484+pub fn is_webcodecs_available(state: State<'_, AppState>) -> bool {485+ state.webcodecs.load(Ordering::SeqCst)486+}487+476488 // ─── Host Registration (keyring) ─────────────────────────────────────────────477489 478490 @@ -14,6 +14,8 @@ fn main() {
1414 commands::fido_pin_retries,1515 commands::titan_derive_identity,1616 commands::is_daemon_mode,17+ commands::set_webcodecs_available,18+ commands::is_webcodecs_available,1719 commands::host_registration_status,1820 commands::titan_register_host,1921 commands::host_unregister,@@ -497,6 +497,11 @@
497497 <div class="info-row"><span class="info-label">pin retries</span><span class="info-value" id="fido-pin">—</span></div>498498 <button class="link-btn" style="margin-top: 8px;" onclick="scanFido()">rescan</button>499499 </div>500+ <div class="panel-section">501+ <h3># codec</h3>502+ <div class="info-row"><span class="info-label">webcodecs</span><span class="info-value" id="codec-webcodecs">checking...</span></div>503+ <div class="info-row"><span class="info-label">decode path</span><span class="info-value" id="codec-path">—</span></div>504+ </div>500505 <div class="panel-section">501506 <h3># registration</h3>502507 <div class="info-row"><span class="info-label">status</span><span class="info-value" id="reg-status">—</span></div>@@ -804,7 +809,52 @@
804809 document.getElementById('host-conn-count').textContent = event.payload;805810 });806811 807- // ─── Frames ─────────────────────────────────────────────────────────812+ // ─── WebCodecs detection ────────────────────────────────────────────813+ let hasWebCodecs = ('VideoDecoder' in window);814+ let videoDecoder = null;815+ let decoderConfigured = false;816+ let pendingKeyframe = true;817+818+ (async function detectWebCodecs() {819+ const el = document.getElementById('codec-webcodecs');820+ const pathEl = document.getElementById('codec-path');821+ if (hasWebCodecs) {822+ el.textContent = 'available';823+ el.style.color = 'var(--green)';824+ pathEl.textContent = 'webcodecs (hw)';825+ await invoke('set_webcodecs_available', { available: true });826+ } else {827+ el.textContent = 'unavailable';828+ el.style.color = 'var(--destructive)';829+ pathEl.textContent = 'openh264 → jpeg';830+ await invoke('set_webcodecs_available', { available: false });831+ }832+ })();833+834+ function initWebCodecsDecoder(width, height) {835+ if (videoDecoder) {836+ try { videoDecoder.close(); } catch(e) {}837+ }838+ videoDecoder = new VideoDecoder({839+ output: (frame) => {840+ ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);841+ frame.close();842+ },843+ error: (e) => {844+ console.error('VideoDecoder error:', e);845+ pendingKeyframe = true;846+ }847+ });848+ videoDecoder.configure({849+ codec: 'avc1.42001e',850+ codedWidth: width,851+ codedHeight: height,852+ });853+ decoderConfigured = true;854+ pendingKeyframe = true;855+ }856+857+ // ─── Frames — dual path: WebCodecs or openh264→JPEG ─────────────────808858 const canvas = document.getElementById('frame-canvas');809859 const ctx = canvas.getContext('2d');810860 @@ -814,15 +864,34 @@
814864 frameHeight = height;815865 if (canvas.width !== width) canvas.width = width;816866 if (canvas.height !== height) canvas.height = height;817- const bytes = Uint8Array.from(atob(data), c => c.charCodeAt(0));818- const blob = new Blob([bytes], { type: 'image/jpeg' });819- const url = URL.createObjectURL(blob);820- const img = new Image();821- img.onload = () => {822- ctx.drawImage(img, 0, 0, canvas.width, canvas.height);823- URL.revokeObjectURL(url);824- };825- img.src = url;867+868+ if (hasWebCodecs) {869+ // Path 1: WebCodecs hardware decode870+ const h264 = Uint8Array.from(atob(data), c => c.charCodeAt(0));871+ if (!decoderConfigured) initWebCodecsDecoder(width, height);872+ const chunk = new EncodedVideoChunk({873+ type: pendingKeyframe ? 'key' : 'delta',874+ timestamp: performance.now() * 1000,875+ data: h264,876+ });877+ try {878+ videoDecoder.decode(chunk);879+ pendingKeyframe = false;880+ } catch (e) {881+ pendingKeyframe = true;882+ }883+ } else {884+ // Path 2: JPEG from backend (openh264 decode → JPEG encode in Rust)885+ const bytes = Uint8Array.from(atob(data), c => c.charCodeAt(0));886+ const blob = new Blob([bytes], { type: 'image/jpeg' });887+ const url = URL.createObjectURL(blob);888+ const img = new Image();889+ img.onload = () => {890+ ctx.drawImage(img, 0, 0, canvas.width, canvas.height);891+ URL.revokeObjectURL(url);892+ };893+ img.src = url;894+ }826895 });827896 828897 listen('frame-stats', (event) => {