openh264 outputs Annex B (start codes), WebCodecs expects AVCC (length-prefixed NALs + description record). Now: - Parse Annex B NAL units from H.264 stream - Extract SPS/PPS from keyframes, build AVCC description for decoder config - Convert slice NALs to 4-byte length-prefixed format - Filter SPS/PPS from decode chunks (already in description) - Reset decoder state on error for recovery on next keyframe
- commit
- 97a9c55eae74e8ebcaa35a136748c12a867d2b5a
- parent
- 944dccf22df891b58c74988c7847e175c08946d6
- author
- dami etoile <dami.etoile@gmail.com>
| M | src/ | 140 | +++++++++++++++++++++++------- |
@@ -830,34 +830,82 @@
830830 }831831 })();832832 833- function parseSpsCodec(h264Data) {834- // Scan for SPS NAL (type 7) and extract profile/level833+ // ─── Annex B → AVCC conversion ─────────────────────────────────────834+ // openh264 outputs Annex B (00 00 00 01 start codes).835+ // WebCodecs expects AVCC (4-byte length-prefixed NALs) + description.836+837+ function parseAnnexBNals(data) {838+ const nals = [];835839 let i = 0;836- while (i < h264Data.length - 4) {837- if (h264Data[i] === 0 && h264Data[i+1] === 0 && h264Data[i+2] === 0 && h264Data[i+3] === 1) {838- const nalType = h264Data[i+4] & 0x1f;839- if (nalType === 7) { // SPS840- const profileIdc = h264Data[i+5];841- const constraintFlags = h264Data[i+6];842- const levelIdc = h264Data[i+7];843- const profileHex = profileIdc.toString(16).padStart(2, '0');844- const constraintHex = constraintFlags.toString(16).padStart(2, '0');845- const levelHex = levelIdc.toString(16).padStart(2, '0');846- return `avc1.${profileHex}${constraintHex}${levelHex}`;847- }848- i += 4;840+ while (i < data.length) {841+ let scl = 0;842+ if (i + 3 < data.length && data[i] === 0 && data[i+1] === 0 && data[i+2] === 0 && data[i+3] === 1) {843+ scl = 4;844+ } else if (i + 2 < data.length && data[i] === 0 && data[i+1] === 0 && data[i+2] === 1) {845+ scl = 3;849846 } else {850- i++;847+ i++; continue;851848 }849+ const nalStart = i + scl;850+ let j = nalStart + 1;851+ while (j < data.length) {852+ if (j + 3 < data.length && data[j] === 0 && data[j+1] === 0 && data[j+2] === 0 && data[j+3] === 1) break;853+ if (j + 2 < data.length && data[j] === 0 && data[j+1] === 0 && data[j+2] === 1) break;854+ j++;855+ }856+ nals.push(data.subarray(nalStart, j));857+ i = j;858+ }859+ return nals;860+ }861+862+ function buildAvcDescription(sps, pps) {863+ const buf = new Uint8Array(11 + sps.length + pps.length);864+ let off = 0;865+ buf[off++] = 1; // configurationVersion866+ buf[off++] = sps[1]; // profile_idc867+ buf[off++] = sps[2]; // constraint_flags868+ buf[off++] = sps[3]; // level_idc869+ buf[off++] = 0xFF; // lengthSizeMinusOne = 3 (4-byte lengths)870+ buf[off++] = 0xE1; // numOfSPS = 1871+ buf[off++] = (sps.length >> 8) & 0xFF;872+ buf[off++] = sps.length & 0xFF;873+ buf.set(sps, off); off += sps.length;874+ buf[off++] = 1; // numOfPPS = 1875+ buf[off++] = (pps.length >> 8) & 0xFF;876+ buf[off++] = pps.length & 0xFF;877+ buf.set(pps, off);878+ return buf.buffer;879+ }880+881+ function nalsToAvcc(nals) {882+ let total = 0;883+ for (const nal of nals) total += 4 + nal.length;884+ const result = new Uint8Array(total);885+ let off = 0;886+ for (const nal of nals) {887+ const dv = new DataView(result.buffer, off, 4);888+ dv.setUint32(0, nal.length, false);889+ result.set(nal, off + 4);890+ off += 4 + nal.length;852891 }853- // Fallback: Baseline, Level 4.0 (supports 1080p)854- return 'avc1.42e028';892+ return result;855893 }856894 857- function initWebCodecsDecoder(width, height, codecStr) {895+ function codecStrFromSps(sps) {896+ const p = sps[1].toString(16).padStart(2, '0');897+ const c = sps[2].toString(16).padStart(2, '0');898+ const l = sps[3].toString(16).padStart(2, '0');899+ return `avc1.${p}${c}${l}`;900+ }901+902+ function initWebCodecsDecoder(width, height, sps, pps) {858903 if (videoDecoder) {859904 try { videoDecoder.close(); } catch(e) {}860905 }906+ const codecStr = codecStrFromSps(sps);907+ const description = buildAvcDescription(sps, pps);908+ console.log('WebCodecs configure:', codecStr, 'w:', width, 'h:', height, 'desc:', description.byteLength, 'bytes');861909 videoDecoder = new VideoDecoder({862910 output: (frame) => {863911 ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);@@ -865,12 +913,16 @@
865913 },866914 error: (e) => {867915 console.error('VideoDecoder error:', e);916+ decoderConfigured = false;917+ try { videoDecoder.close(); } catch(_) {}918+ videoDecoder = null;868919 }869920 });870921 videoDecoder.configure({871922 codec: codecStr,872923 codedWidth: width,873924 codedHeight: height,925+ description: description,874926 optimizeForRealtimeUse: true,875927 });876928 decoderConfigured = true;@@ -890,20 +942,44 @@
890942 if (hasWebCodecs) {891943 // Path 1: WebCodecs hardware decode892944 const h264 = Uint8Array.from(atob(data), c => c.charCodeAt(0));893- if (!decoderConfigured) {894- const codecStr = parseSpsCodec(h264);895- console.log('WebCodecs codec:', codecStr, 'w:', width, 'h:', height);896- initWebCodecsDecoder(width, height, codecStr);945+ const nals = parseAnnexBNals(h264);946+947+ // Extract SPS/PPS from keyframes to (re)configure decoder948+ if (event.payload.keyframe) {949+ let sps = null, pps = null;950+ for (const nal of nals) {951+ const nalType = nal[0] & 0x1f;952+ if (nalType === 7) sps = nal;953+ else if (nalType === 8) pps = nal;954+ }955+ if (sps && pps) {956+ if (!decoderConfigured) {957+ initWebCodecsDecoder(width, height, sps, pps);958+ }959+ }897960 }898- const chunk = new EncodedVideoChunk({899- type: event.payload.keyframe ? 'key' : 'delta',900- timestamp: performance.now() * 1000,901- data: h264,902- });903- try {904- videoDecoder.decode(chunk);905- } catch (e) {906- console.warn('decode failed:', e);961+962+ if (!decoderConfigured || !videoDecoder) {963+ // Waiting for first keyframe with SPS/PPS964+ } else {965+ // Filter out SPS/PPS — already in description, decoder doesn't want them again966+ const sliceNals = nals.filter(nal => {967+ const t = nal[0] & 0x1f;968+ return t !== 7 && t !== 8; // skip SPS, PPS969+ });970+ if (sliceNals.length > 0) {971+ const avcc = nalsToAvcc(sliceNals);972+ const chunk = new EncodedVideoChunk({973+ type: event.payload.keyframe ? 'key' : 'delta',974+ timestamp: performance.now() * 1000,975+ data: avcc,976+ });977+ try {978+ videoDecoder.decode(chunk);979+ } catch (e) {980+ console.warn('decode failed:', e);981+ }982+ }907983 }908984 } else {909985 // Path 2: JPEG from backend (openh264 decode → JPEG encode in Rust)