Frontend now handles all three codecs for WebCodecs hardware decode:
H.264 (avc1):
- SPS+PPS → AVCC description (unchanged)
- NAL type 7/8 filtered from decode chunks (also type 9 AUD)
H.265 (hvc1):
- VPS+SPS+PPS → hvcC description (ISO 14496-15)
- HEVC NAL types: VPS(32), SPS(33), PPS(34), AUD(35) filtered
- NAL type extracted with (byte>>1)&0x3f (2-byte HEVC header)
- Codec string: hvc1.{profile}.4.L{level}.B0
AV1 (av01):
- Sequence header OBU (type 12) → raw bytes as description
- OBU parser handles leb128 size fields
- TD (type 2) and seq header filtered from decode chunks
- Codec string: av01.{profile}.{level}M.08
Backend emits host-codec event when streaming starts so the
frontend sets activeCodec before frames arrive. Changing codec
in the encoder config panel resets the decoder for the new codec.- commit
- 2bd6aee835dd1f0482ef90efa2651fd3c820717d
- parent
- 695a1df6221af2c62fa8c2070ac69785c9efa3d7
- author
- dami etoile <dami.etoile@gmail.com>
| M | src-tauri/ | 3 | + |
| M | src/ | 333 | +++++++++++++++++++++++++----- |
@@ -979,6 +979,9 @@ async fn stream_frames_ffmpeg(
979979 encoder, fmt, bsf, config.bitrate, config.framerate, config.gop980980 );981981 982+ // Notify frontend of active codec for decoder configuration983+ let _ = app.emit("host-codec", &config.codec);984+982985 let mut cmd = TokioCommand::new("ffmpeg");983986 cmd.stdin(Stdio::null())984987 .stdout(Stdio::piped())@@ -979,9 +979,12 @@
979979 await invoke('set_encoder_config', { config });980980 }981981 982- // ─── Annex B → AVCC conversion ─────────────────────────────────────983- // openh264 outputs Annex B (00 00 00 01 start codes).984- // WebCodecs expects AVCC (4-byte length-prefixed NALs) + description.982+ // ─── Annex B → AVCC/hvcC/av1C conversion ────────────────────────────983+ // ffmpeg outputs Annex B (00 00 00 01 start codes) for H.264/H.265.984+ // WebCodecs expects length-prefixed NALs + description record.985+ // AV1 uses OBU format directly.986+987+ let activeCodec = 'h264'; // updated from host-codec event985988 986989 function parseAnnexBNals(data) {987990 const nals = [];@@ -1008,6 +1011,22 @@
10081011 return nals;10091012 }10101013 1014+ function nalsToLengthPrefixed(nals) {1015+ let total = 0;1016+ for (const nal of nals) total += 4 + nal.length;1017+ const result = new Uint8Array(total);1018+ let off = 0;1019+ for (const nal of nals) {1020+ const dv = new DataView(result.buffer, off, 4);1021+ dv.setUint32(0, nal.length, false);1022+ result.set(nal, off + 4);1023+ off += 4 + nal.length;1024+ }1025+ return result;1026+ }1027+1028+ // ─── H.264 ─────────────────────────────────────────────────────────1029+10111030 function buildAvcDescription(sps, pps) {10121031 const buf = new Uint8Array(11 + sps.length + pps.length);10131032 let off = 0;@@ -1015,7 +1034,7 @@
10151034 buf[off++] = sps[1]; // profile_idc10161035 buf[off++] = sps[2]; // constraint_flags10171036 buf[off++] = sps[3]; // level_idc1018- buf[off++] = 0xFF; // lengthSizeMinusOne = 3 (4-byte lengths)1037+ buf[off++] = 0xFF; // lengthSizeMinusOne = 310191038 buf[off++] = 0xE1; // numOfSPS = 110201039 buf[off++] = (sps.length >> 8) & 0xFF;10211040 buf[off++] = sps.length & 0xFF;@@ -1027,34 +1046,167 @@
10271046 return buf.buffer;10281047 }10291048 1030- function nalsToAvcc(nals) {1031- let total = 0;1032- for (const nal of nals) total += 4 + nal.length;1033- const result = new Uint8Array(total);1034- let off = 0;1035- for (const nal of nals) {1036- const dv = new DataView(result.buffer, off, 4);1037- dv.setUint32(0, nal.length, false);1038- result.set(nal, off + 4);1039- off += 4 + nal.length;1040- }1041- return result;1042- }1043-1044- function codecStrFromSps(sps) {1049+ function avcCodecStr(sps) {10451050 const p = sps[1].toString(16).padStart(2, '0');10461051 const c = sps[2].toString(16).padStart(2, '0');10471052 const l = sps[3].toString(16).padStart(2, '0');10481053 return `avc1.${p}${c}${l}`;10491054 }10501055 1051- function initWebCodecsDecoder(width, height, sps, pps) {1056+ function h264NalType(nal) { return nal[0] & 0x1f; }1057+1058+ // ─── H.265 / HEVC ──────────────────────────────────────────────────1059+1060+ function buildHvcDescription(vps, sps, pps) {1061+ // hvcC record format (ISO 14496-15 8.3.3.1.2)1062+ const buf = new Uint8Array(23 + 3 + vps.length + sps.length + pps.length);1063+ let off = 0;1064+ buf[off++] = 1; // configurationVersion1065+ // profile space(2) + tier(1) + profile_idc(5)1066+ buf[off++] = sps[1] & 0x1f; // profile_space=0, tier=0, profile_idc from SPS1067+ buf[off++] = sps[2]; // general_profile_compatibility_flags[0..7]1068+ buf[off++] = sps[3]; // [8..15]1069+ buf[off++] = sps[4]; // [16..23]1070+ buf[off++] = sps[5]; // [24..31]1071+ // constraint flags (6 bytes)1072+ for (let i = 0; i < 6; i++) buf[off++] = sps[6 + i] || 0;1073+ buf[off++] = sps[12]; // general_level_idc1074+ // min_spatial_segmentation_idc (12 bits, 0xF000 = reserved)1075+ buf[off++] = 0xF0;1076+ buf[off++] = 0x00;1077+ // parallelismType (2 bits, 0xFC = reserved)1078+ buf[off++] = 0xFC;1079+ // chromaFormat (2 bits, 0xFC = reserved) — from SPS1080+ buf[off++] = 0xFC | ((sps.length > 2 ? 0 : 0));1081+ // bitDepthLumaMinus8 (3 bits, 0xF8 = reserved)1082+ buf[off++] = 0xF8;1083+ // bitDepthChromaMinus8 (3 bits, 0xF8 = reserved)1084+ buf[off++] = 0xF8;1085+ // avgFrameRate1086+ buf[off++] = 0x00;1087+ buf[off++] = 0x00;1088+ // constantFrameRate(2) + numTemporalLayers(3) + temporalIdNested(1) + lengthSizeMinusOne(2)1089+ buf[off++] = 0x0F; // lengthSizeMinusOne=31090+ // numOfArrays1091+ buf[off++] = 3; // VPS, SPS, PPS1092+1093+ // Helper to write a NAL array entry1094+ const writeNal = (nalType, nal) => {1095+ buf[off++] = 0x80 | nalType; // array_completeness=11096+ buf[off++] = 0; // numNalus high byte1097+ buf[off++] = 1; // numNalus = 11098+ buf[off++] = (nal.length >> 8) & 0xFF;1099+ buf[off++] = nal.length & 0xFF;1100+ buf.set(nal, off);1101+ off += nal.length;1102+ };1103+1104+ writeNal(32, vps); // VPS1105+ writeNal(33, sps); // SPS1106+ writeNal(34, pps); // PPS1107+1108+ return buf.buffer;1109+ }1110+1111+ function hevcCodecStr(sps) {1112+ // General profile space (2 bits) = 0, tier (1 bit) = 0 (Main tier)1113+ const profileIdc = (sps[1] & 0x1f);1114+ // General level idc is at byte 12 in the SPS NAL (after the NAL header byte)1115+ // For HEVC, NAL header is 2 bytes: byte[0] = (forbidden<<7 | nal_type<<1 | nuh_layer_id>>5)1116+ // byte[1] = (nuh_layer_id<<3 | nuh_temporal_id_plus1)1117+ // So SPS data starts at byte 21118+ const levelIdc = sps.length > 13 ? sps[13] : 0;1119+ // Simplified: use profile_idc from SPS, level from SPS byte 131120+ // Format: hev1.X.Y.L where X=profile_space(0), Y=tier(0 for Main), L=level1121+ const p = profileIdc.toString(16).padStart(2, '0');1122+ const l = levelIdc.toString(16).padStart(2, '0');1123+ // WebCodecs uses 'hev1' or 'hvc1' — 'hev1' is the parameter-set-in-band variant1124+ // Use 'hvc1' since we provide description1125+ return `hvc1.${p}.4.L${levelIdc}.B0`;1126+ }1127+1128+ function hevcNalType(nal) { return (nal[0] >> 1) & 0x3f; }1129+1130+ // ─── AV1 ───────────────────────────────────────────────────────────1131+1132+ // AV1 uses OBU format, no start codes. The sequence header OBU1133+ // serves as the description. WebCodecs accepts the raw sequence header1134+ // bytes as the description buffer.1135+1136+ function parseAv1Obus(data) {1137+ // AV1 low-overhead bitstream format: each OBU is length-prefixed1138+ // With temporal delimiter (td=insert), each frame starts with a TD OBU1139+ const obus = [];1140+ let i = 0;1141+ while (i < data.length) {1142+ if (i + 1 >= data.length) break;1143+ const header = data[i];1144+ const obuType = (header >> 3) & 0x0f;1145+ const hasExtension = (header & 0x04) !== 0;1146+ const hasSize = (header & 0x02) !== 0;1147+ let off = i + 1 + (hasExtension ? 1 : 0);1148+ if (!hasSize) {1149+ // Rest of data is this OBU1150+ obus.push({ type: obuType, data: data.subarray(i, data.length) });1151+ break;1152+ }1153+ // Read leb128 size1154+ let size = 0;1155+ let shift = 0;1156+ let bytesConsumed = 0;1157+ for (let j = 0; j < 4 && off + j < data.length; j++) {1158+ const b = data[off + j];1159+ size |= (b & 0x7f) << shift;1160+ shift += 7;1161+ bytesConsumed++;1162+ if ((b & 0x80) === 0) break;1163+ }1164+ off += bytesConsumed;1165+ obus.push({ type: obuType, data: data.subarray(i, off + size) });1166+ i = off + size;1167+ }1168+ return obus;1169+ }1170+1171+ function av1CodecStr(sequenceHeader) {1172+ // Parse the AV1 sequence header to build codec string1173+ // seq_profile (3 bits), level (5 bits)1174+ // For 'av01.P.LLT.MMM' format1175+ const obuData = sequenceHeader;1176+ // Skip OBU header byte(s) to get to sequence_header_obu data1177+ let off = 1; // obu_header byte1178+ // Check for extension flag1179+ if ((obuData[0] & 0x04) !== 0) off = 2;1180+ // Skip size leb1281181+ while (off < obuData.length && (obuData[off] & 0x80) !== 0) off++;1182+ off++;1183+ // Now at sequence_header_obu data1184+ // seq_profile (3 bits) + still_picture (1) + reduced_still_picture_header (1)1185+ const b0 = obuData[off] || 0;1186+ const profile = (b0 >> 5) & 0x07;1187+ const reduced = (b0 >> 3) & 0x01;1188+ let levelIdx;1189+ if (reduced) {1190+ levelIdx = (b0 >> 2) & 0x1f;1191+ } else {1192+ // More complex parsing needed; use defaults1193+ levelIdx = 0;1194+ }1195+ return `av01.${profile}.${levelIdx.toString().padStart(2, '0')}M.08`;1196+ }1197+1198+ function buildAv1Description(seqHeaderObu) {1199+ // WebCodecs accepts the raw sequence header OBU bytes1200+ return seqHeaderObu.buffer.slice(seqHeaderObu.byteOffset, seqHeaderObu.byteOffset + seqHeaderObu.byteLength);1201+ }1202+1203+ // ─── WebCodecs init per codec ──────────────────────────────────────1204+1205+ function initWebCodecsDecoder(width, height, desc, codecStr) {10521206 if (videoDecoder) {10531207 try { videoDecoder.close(); } catch(e) {}10541208 }1055- const codecStr = codecStrFromSps(sps);1056- const description = buildAvcDescription(sps, pps);1057- console.log('WebCodecs configure:', codecStr, 'w:', width, 'h:', height, 'desc:', description.byteLength, 'bytes');1209+ console.log('WebCodecs configure:', codecStr, 'w:', width, 'h:', height, 'desc:', desc.byteLength, 'bytes');10581210 videoDecoder = new VideoDecoder({10591211 output: (frame) => {10601212 ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);@@ -1071,16 +1223,27 @@
10711223 codec: codecStr,10721224 codedWidth: width,10731225 codedHeight: height,1074- description: description,1226+ description: desc,10751227 optimizeForRealtimeUse: true,10761228 });10771229 decoderConfigured = true;10781230 }10791231 1080- // ─── Frames — dual path: WebCodecs or openh264→JPEG ─────────────────1232+ // ─── Frames — multi-codec WebCodecs or JPEG fallback ───────────────10811233 const canvas = document.getElementById('frame-canvas');10821234 const ctx = canvas.getContext('2d');10831235 1236+ listen('host-codec', (event) => {1237+ activeCodec = event.payload || 'h264';1238+ console.log('host codec:', activeCodec);1239+ // Reset decoder — new codec needs new config1240+ decoderConfigured = false;1241+ if (videoDecoder) {1242+ try { videoDecoder.close(); } catch(_) {}1243+ videoDecoder = null;1244+ }1245+ });1246+10841247 listen('frame', (event) => {10851248 const { width, height, data } = event.payload;10861249 frameWidth = width;@@ -1089,46 +1252,112 @@
10891252 if (canvas.height !== height) canvas.height = height;10901253 10911254 if (hasWebCodecs) {1092- // Path 1: WebCodecs hardware decode1093- const h264 = Uint8Array.from(atob(data), c => c.charCodeAt(0));1094- const nals = parseAnnexBNals(h264);1095-1096- // Extract SPS/PPS from keyframes to (re)configure decoder1097- if (event.payload.keyframe) {1098- let sps = null, pps = null;1099- for (const nal of nals) {1100- const nalType = nal[0] & 0x1f;1101- if (nalType === 7) sps = nal;1102- else if (nalType === 8) pps = nal;1255+ const raw = Uint8Array.from(atob(data), c => c.charCodeAt(0));1256+1257+ if (activeCodec === 'av1') {1258+ // ── AV1 path ──1259+ const obus = parseAv1Obus(raw);1260+1261+ if (event.payload.keyframe) {1262+ // Find sequence header OBU (type 12)1263+ const seqHeader = obus.find(o => o.type === 12);1264+ if (seqHeader && !decoderConfigured) {1265+ const desc = buildAv1Description(seqHeader.data);1266+ const codecStr = av1CodecStr(seqHeader.data);1267+ initWebCodecsDecoder(width, height, desc, codecStr);1268+ }11031269 }1104- if (sps && pps) {1105- if (!decoderConfigured) {1106- initWebCodecsDecoder(width, height, sps, pps);1270+1271+ if (!decoderConfigured || !videoDecoder) return;1272+1273+ // Filter out TD and sequence header OBUs — already in description1274+ const frameObus = obus.filter(o => o.type !== 2 && o.type !== 12);1275+ if (frameObus.length > 0) {1276+ // Concatenate remaining OBUs as the chunk data1277+ let totalLen = 0;1278+ for (const o of frameObus) totalLen += o.data.length;1279+ const chunkData = new Uint8Array(totalLen);1280+ let off = 0;1281+ for (const o of frameObus) { chunkData.set(o.data, off); off += o.data.length; }1282+1283+ const chunk = new EncodedVideoChunk({1284+ type: event.payload.keyframe ? 'key' : 'delta',1285+ timestamp: performance.now() * 1000,1286+ data: chunkData,1287+ });1288+ try { videoDecoder.decode(chunk); } catch (e) { console.warn('av1 decode failed:', e); }1289+ }1290+1291+ } else if (activeCodec === 'h265') {1292+ // ── H.265 path ──1293+ const nals = parseAnnexBNals(raw);1294+1295+ if (event.payload.keyframe) {1296+ let vps = null, sps = null, pps = null;1297+ for (const nal of nals) {1298+ const t = hevcNalType(nal);1299+ if (t === 32) vps = nal;1300+ else if (t === 33) sps = nal;1301+ else if (t === 34) pps = nal;1302+ }1303+ if (vps && sps && pps && !decoderConfigured) {1304+ const desc = buildHvcDescription(vps, sps, pps);1305+ const codecStr = hevcCodecStr(sps);1306+ initWebCodecsDecoder(width, height, desc, codecStr);11071307 }11081308 }1109- }11101309 1111- if (!decoderConfigured || !videoDecoder) {1112- // Waiting for first keyframe with SPS/PPS1113- } else {1114- // Filter out SPS/PPS — already in description, decoder doesn't want them again1310+ if (!decoderConfigured || !videoDecoder) return;1311+1312+ // Filter out VPS/SPS/PPS/AUD — already in description11151313 const sliceNals = nals.filter(nal => {1116- const t = nal[0] & 0x1f;1117- return t !== 7 && t !== 8; // skip SPS, PPS1314+ const t = hevcNalType(nal);1315+ return t !== 32 && t !== 33 && t !== 34 && t !== 35;11181316 });11191317 if (sliceNals.length > 0) {1120- const avcc = nalsToAvcc(sliceNals);1318+ const lpData = nalsToLengthPrefixed(sliceNals);11211319 const chunk = new EncodedVideoChunk({11221320 type: event.payload.keyframe ? 'key' : 'delta',11231321 timestamp: performance.now() * 1000,1124- data: avcc,1322+ data: lpData,11251323 });1126- try {1127- videoDecoder.decode(chunk);1128- } catch (e) {1129- console.warn('decode failed:', e);1324+ try { videoDecoder.decode(chunk); } catch (e) { console.warn('hevc decode failed:', e); }1325+ }1326+1327+ } else {1328+ // ── H.264 path (default) ──1329+ const nals = parseAnnexBNals(raw);1330+1331+ if (event.payload.keyframe) {1332+ let sps = null, pps = null;1333+ for (const nal of nals) {1334+ const t = h264NalType(nal);1335+ if (t === 7) sps = nal;1336+ else if (t === 8) pps = nal;1337+ }1338+ if (sps && pps && !decoderConfigured) {1339+ const desc = buildAvcDescription(sps, pps);1340+ const codecStr = avcCodecStr(sps);1341+ initWebCodecsDecoder(width, height, desc, codecStr);11301342 }11311343 }1344+1345+ if (!decoderConfigured || !videoDecoder) return;1346+1347+ // Filter out SPS/PPS/AUD — already in description1348+ const sliceNals = nals.filter(nal => {1349+ const t = h264NalType(nal);1350+ return t !== 7 && t !== 8 && t !== 9;1351+ });1352+ if (sliceNals.length > 0) {1353+ const lpData = nalsToLengthPrefixed(sliceNals);1354+ const chunk = new EncodedVideoChunk({1355+ type: event.payload.keyframe ? 'key' : 'delta',1356+ timestamp: performance.now() * 1000,1357+ data: lpData,1358+ });1359+ try { videoDecoder.decode(chunk); } catch (e) { console.warn('h264 decode failed:', e); }1360+ }11321361 }11331362 } else {11341363 // Path 2: JPEG from backend (openh264 decode → JPEG encode in Rust)