Remove unreachable, spec-invalid HEVC and AV1 WebCodecs builders. Reject unsupported legacy and binary channel codec IDs before payload allocation or decoder construction, with focused Rust and frontend coverage.
- commit
- 0db73cf2aa7233277ef168f1889ebefc341dcaf1
- parent
- 2cb5380110896da0b8f085a01b0d337a051f4c90
- author
- FelineStateMachine <dami.etoile@gmail.com>
| M | portal/ | 102 | +--------------------------- |
| M | portal/ | 2 | +- |
| M | portal/ | 2 | + |
| M | portal/ | 2 | +- |
| M | portal/ | 7 | ++ |
| M | portal/ | 109 | ++++++------------------------ |
| M | portal/ | 103 | ++++++---------------------- |
| M | src-tauri/ | 27 | +++++-- |
| M | src-tauri/ | 8 | ++- |
@@ -1,7 +1,6 @@
1-// ─── Annex B → AVCC/hvcC/av1C conversion ──────────────────────────────────2-// ffmpeg outputs Annex B (00 00 00 01 start codes) for H.264/H.265.3-// WebCodecs expects length-prefixed NALs + description record.4-// AV1 uses OBU format directly.1+// ─── H.264 Annex B → AVCC conversion ──────────────────────────────────────2+// Sigil's negotiated media contract is H.264-only. The encoder emits Annex B3+// start codes; WebCodecs expects length-prefixed NALs plus an AVC description.54 65 export function parseAnnexBNals(data) {76 const nals = [];@@ -71,98 +70,3 @@ export function avcCodecStr(sps) {
7170 const l = sps[3].toString(16).padStart(2, '0');7271 return `avc1.${p}${c}${l}`;7372 }74-75-// ─── H.265 / HEVC ──────────────────────────────────────────────────────────76-77-export function hevcNalType(nal) { return (nal[0] >> 1) & 0x3f; }78-79-export function buildHvcDescription(vps, sps, pps) {80- // hvcC record format (ISO 14496-15 8.3.3.1.2)81- const buf = new Uint8Array(23 + 15 + vps.length + sps.length + pps.length);82- let off = 0;83- buf[off++] = 1; // configurationVersion84- buf[off++] = sps[1] & 0x1f; // profile_space=0, tier=0, profile_idc from SPS85- buf[off++] = sps[2]; // general_profile_compatibility_flags[0..7]86- buf[off++] = sps[3]; // [8..15]87- buf[off++] = sps[4]; // [16..23]88- buf[off++] = sps[5]; // [24..31]89- for (let i = 0; i < 6; i++) buf[off++] = sps[6 + i] || 0; // constraint flags90- buf[off++] = sps[12]; // general_level_idc91- buf[off++] = 0xF0; buf[off++] = 0x00; // min_spatial_segmentation_idc92- buf[off++] = 0xFC; // parallelismType93- buf[off++] = 0xFC | ((sps.length > 2 ? 0 : 0)); // chromaFormat94- buf[off++] = 0xF8; // bitDepthLumaMinus895- buf[off++] = 0xF8; // bitDepthChromaMinus896- buf[off++] = 0x00; buf[off++] = 0x00; // avgFrameRate97- buf[off++] = 0x0F; // lengthSizeMinusOne=398- buf[off++] = 3; // numOfArrays: VPS, SPS, PPS99-100- const writeNal = (nalType, nal) => {101- buf[off++] = 0x80 | nalType; // array_completeness=1102- buf[off++] = 0;103- buf[off++] = 1; // numNalus = 1104- buf[off++] = (nal.length >> 8) & 0xFF;105- buf[off++] = nal.length & 0xFF;106- buf.set(nal, off);107- off += nal.length;108- };109- writeNal(32, vps);110- writeNal(33, sps);111- writeNal(34, pps);112-113- return buf.buffer;114-}115-116-export function hevcCodecStr(sps) {117- const profileIdc = (sps[1] & 0x1f);118- const levelIdc = sps.length > 13 ? sps[13] : 0;119- return `hvc1.${profileIdc.toString(16).padStart(2, '0')}.4.L${levelIdc}.B0`;120-}121-122-// ─── AV1 ───────────────────────────────────────────────────────────────────123-124-export function parseAv1Obus(data) {125- // AV1 low-overhead bitstream: each OBU is length-prefixed126- const obus = [];127- let i = 0;128- while (i < data.length) {129- if (i + 1 >= data.length) break;130- const header = data[i];131- const obuType = (header >> 3) & 0x0f;132- const hasExtension = (header & 0x04) !== 0;133- const hasSize = (header & 0x02) !== 0;134- let off = i + 1 + (hasExtension ? 1 : 0);135- if (!hasSize) {136- obus.push({ type: obuType, data: data.subarray(i, data.length) });137- break;138- }139- let size = 0, shift = 0, bytesConsumed = 0;140- for (let j = 0; j < 4 && off + j < data.length; j++) {141- const b = data[off + j];142- size |= (b & 0x7f) << shift;143- shift += 7;144- bytesConsumed++;145- if ((b & 0x80) === 0) break;146- }147- off += bytesConsumed;148- obus.push({ type: obuType, data: data.subarray(i, off + size) });149- i = off + size;150- }151- return obus;152-}153-154-export function av1CodecStr(sequenceHeader) {155- const obuData = sequenceHeader;156- let off = (obuData[0] & 0x04) !== 0 ? 2 : 1;157- while (off < obuData.length && (obuData[off] & 0x80) !== 0) off++;158- off++;159- const b0 = obuData[off] || 0;160- const profile = (b0 >> 5) & 0x07;161- const reduced = (b0 >> 3) & 0x01;162- const levelIdx = reduced ? (b0 >> 2) & 0x1f : 0;163- return `av01.${profile}.${levelIdx.toString().padStart(2, '0')}M.08`;164-}165-166-export function buildAv1Description(seqHeaderObu) {167- return seqHeaderObu.buffer.slice(seqHeaderObu.byteOffset, seqHeaderObu.byteOffset + seqHeaderObu.byteLength);168-}@@ -12,7 +12,7 @@ const CODEC_CONFIG_FLAG = 1 << 2;
1212 const OPTIONAL_U64_NONE = 0xffffffffffffffffn;1313 const OPTIONAL_I64_NONE = -0x8000000000000000n;1414 const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER);15-const CODECS = Object.freeze({ 1: 'h264', 2: 'h265', 3: 'av1' });15+const CODECS = Object.freeze({ 1: 'h264' });1616 1717 function exactOptionalUnsigned(value, field) {1818 if (value === OPTIONAL_U64_NONE) return null;@@ -80,6 +80,8 @@ test('rejects malformed identity, version, codec, flags, and reserved fields', (
8080 (bytes) => { bytes[0] = 0; },8181 (bytes) => { bytes[4] = 2; },8282 (bytes) => { bytes[5] = 0; },83+ (bytes) => { bytes[5] = 2; },84+ (bytes) => { bytes[5] = 3; },8385 (bytes) => { bytes[6] = 0x80; },8486 (bytes) => { bytes[6] = 0b100; },8587 (bytes) => { bytes[7] = 1; },@@ -1,5 +1,5 @@
11 function validFormat({ codec, width, height }) {2- if (!['h264', 'h265', 'av1'].includes(codec)) throw new TypeError('invalid video codec');2+ if (codec !== 'h264') throw new TypeError('invalid video codec');33 if (!Number.isSafeInteger(width) || width <= 0 || !Number.isSafeInteger(height) || height <= 0) {44 throw new TypeError('invalid video dimensions');55 }@@ -55,3 +55,10 @@ test('a discontinuity starts a new configured epoch even at the same dimensions'
5555 assert.equal(boundary.reconfigure, true);5656 assert.equal(boundary.epoch, 2);5757 });58+59+test('only the negotiated H.264 codec is accepted', () => {60+ const guard = new VideoFormatTransitionGuard();61+ assert.throws(() => guard.plan(frame({ codec: 'h265' })), /invalid video codec/);62+ assert.throws(() => guard.plan(frame({ codec: 'av1' })), /invalid video codec/);63+ assert.equal(guard.format, null);64+});@@ -4,12 +4,6 @@ import {
44 buildAvcDescription,55 avcCodecStr,66 h264NalType,7- buildHvcDescription,8- hevcCodecStr,9- hevcNalType,10- parseAv1Obus,11- av1CodecStr,12- buildAv1Description,137 } from './codecs.js';148 import { exactMediaTimestampMicros } from './av-sync.mjs';159 import {@@ -349,90 +343,31 @@ export function createVideoPipelineSession({
349343 ? mediaPtsMicros350344 : now() * 1000;351345 352- if (codec === 'av1') {353- // ── AV1 path ──354- const obus = parseAv1Obus(raw);355- if (keyframe) {356- const seqHeader = obus.find(o => o.type === 12);357- if (seqHeader && formatPlan.reconfigure) {358- initWebCodecsDecoder(width, height, buildAv1Description(seqHeader.data), av1CodecStr(seqHeader.data));359- }360- }361- if (!decoderConfigured || !videoDecoder) { droppedFrames++; return; }362- const frameObus = obus.filter(o => o.type !== 2 && o.type !== 12);363- if (frameObus.length > 0) {364- let totalLen = 0;365- for (const o of frameObus) totalLen += o.data.length;366- const chunkData = new Uint8Array(totalLen);367- let off = 0;368- for (const o of frameObus) { chunkData.set(o.data, off); off += o.data.length; }369- const chunk = createEncodedChunk({370- type: keyframe ? 'key' : 'delta',371- timestamp: chunkTimestamp,372- data: chunkData,373- });374- const enqueued = enqueueVideoChunk(chunk, receivedAt, 'av1', mediaPtsMicros);375- commitVideoFrame(formatPlan, keyframe, enqueued);376- }377-378- } else if (codec === 'h265') {379- // ── H.265 path ──380- const nals = parseAnnexBNals(raw);381- if (keyframe) {382- let vps = null, sps = null, pps = null;383- for (const nal of nals) {384- const type = hevcNalType(nal);385- if (type === 32) vps = nal;386- else if (type === 33) sps = nal;387- else if (type === 34) pps = nal;388- }389- if (vps && sps && pps && formatPlan.reconfigure) {390- initWebCodecsDecoder(width, height, buildHvcDescription(vps, sps, pps), hevcCodecStr(sps));391- }392- }393- if (!decoderConfigured || !videoDecoder) { droppedFrames++; return; }394- const sliceNals = nals.filter(nal => {395- const type = hevcNalType(nal);396- return type !== 32 && type !== 33 && type !== 34 && type !== 35;397- });398- if (sliceNals.length > 0) {399- const chunk = createEncodedChunk({400- type: keyframe ? 'key' : 'delta',401- timestamp: chunkTimestamp,402- data: nalsToLengthPrefixed(sliceNals),403- });404- const enqueued = enqueueVideoChunk(chunk, receivedAt, 'hevc', mediaPtsMicros);405- commitVideoFrame(formatPlan, keyframe, enqueued);346+ const nals = parseAnnexBNals(raw);347+ if (keyframe) {348+ let sps = null, pps = null;349+ for (const nal of nals) {350+ const type = h264NalType(nal);351+ if (type === 7) sps = nal;352+ else if (type === 8) pps = nal;406353 }407-408- } else {409- // ── H.264 path (default) ──410- const nals = parseAnnexBNals(raw);411- if (keyframe) {412- let sps = null, pps = null;413- for (const nal of nals) {414- const type = h264NalType(nal);415- if (type === 7) sps = nal;416- else if (type === 8) pps = nal;417- }418- if (sps && pps && formatPlan.reconfigure) {419- initWebCodecsDecoder(width, height, buildAvcDescription(sps, pps), avcCodecStr(sps));420- }354+ if (sps && pps && formatPlan.reconfigure) {355+ initWebCodecsDecoder(width, height, buildAvcDescription(sps, pps), avcCodecStr(sps));421356 }422- if (!decoderConfigured || !videoDecoder) { droppedFrames++; return; }423- const sliceNals = nals.filter(nal => {424- const type = h264NalType(nal);425- return type !== 7 && type !== 8 && type !== 9;357+ }358+ if (!decoderConfigured || !videoDecoder) { droppedFrames++; return; }359+ const sliceNals = nals.filter(nal => {360+ const type = h264NalType(nal);361+ return type !== 7 && type !== 8 && type !== 9;362+ });363+ if (sliceNals.length > 0) {364+ const chunk = createEncodedChunk({365+ type: keyframe ? 'key' : 'delta',366+ timestamp: chunkTimestamp,367+ data: nalsToLengthPrefixed(sliceNals),426368 });427- if (sliceNals.length > 0) {428- const chunk = createEncodedChunk({429- type: keyframe ? 'key' : 'delta',430- timestamp: chunkTimestamp,431- data: nalsToLengthPrefixed(sliceNals),432- });433- const enqueued = enqueueVideoChunk(chunk, receivedAt, 'h264', mediaPtsMicros);434- commitVideoFrame(formatPlan, keyframe, enqueued);435- }369+ const enqueued = enqueueVideoChunk(chunk, receivedAt, 'h264', mediaPtsMicros);370+ commitVideoFrame(formatPlan, keyframe, enqueued);436371 }437372 } else {438373 // JPEG fallback (openh264 decode → JPEG encode in Rust)@@ -18,20 +18,6 @@ const h264Delta = () => new Uint8Array([
1818 0, 0, 0, 1, 0x41, 0x9a,1919 ]);2020 21-const h265Keyframe = () => new Uint8Array([22- 0, 0, 0, 1, 0x40, 0x01, 0x0c,23- 0, 0, 0, 1, 0x42, 0x01, 0x01, 0x60, 0, 0, 0, 0, 0, 0, 0, 0, 0x5d, 0x78,24- 0, 0, 0, 1, 0x44, 0x01, 0xc0,25- 0, 0, 0, 1, 0x46, 0x01,26- 0, 0, 0, 1, 0x26, 0x01, 0xaa,27-]);28-29-const av1Keyframe = () => new Uint8Array([30- 0x62, 0x02, 0x20, 0x00,31- 0x12, 0x00,32- 0x32, 0x02, 0xaa, 0xbb,33-]);34-3521 function harness({ hasWebCodecs = true } = {}) {3622 let clock = 10;3723 const decoders = [];@@ -151,73 +137,28 @@ test('configured H.264 keyframes commit format only after decode enqueue', () =>
151137 assert.equal(stats.decoderQueueCapacity, MAX_DECODE_QUEUE_SIZE);152138 });153139 154-test('configured H.265 keyframes keep parameter sets out of the decoded chunk', () => {155- const subject = harness();156- subject.pipeline.processFramePayload({157- width: 1920,158- height: 1080,159- codec: 'h265',160- keyframe: true,161- codecConfig: true,162- sequence: 1,163- pts_micros: 2_000,164- discontinuity: false,165- data: null,166- }, h265Keyframe());167-168- assert.equal(subject.decoders.length, 1);169- assert.deepEqual(subject.decoders[0].configureCalls[0], {170- codec: 'hvc1.01.4.L120.B0',171- codedWidth: 1920,172- codedHeight: 1080,173- description: subject.decoders[0].configureCalls[0].description,174- optimizeForLatency: true,175- });176- assert.equal(subject.decoders[0].configureCalls[0].description.byteLength, 58);177- assert.deepEqual(subject.pipeline.format, {178- codec: 'h265', width: 1920, height: 1080, epoch: 1,179- });180- assert.deepEqual(subject.formats, [subject.pipeline.format]);181- assert.equal(subject.chunks.length, 1);182- assert.equal(subject.chunks[0].type, 'key');183- assert.equal(subject.chunks[0].timestamp, 2_000);184- assert.deepEqual([...subject.chunks[0].data], [0, 0, 0, 3, 0x26, 0x01, 0xaa]);185-});186-187-test('configured AV1 keyframes keep sequence and temporal headers out of the decoded chunk', () => {188- const subject = harness();189- subject.pipeline.processFramePayload({190- width: 1280,191- height: 720,192- codec: 'av1',193- keyframe: true,194- codecConfig: true,195- sequence: 1,196- pts_micros: 3_000,197- discontinuity: false,198- data: null,199- }, av1Keyframe());200-201- assert.equal(subject.decoders.length, 1);202- assert.deepEqual(subject.decoders[0].configureCalls[0], {203- codec: 'av01.1.00M.08',204- codedWidth: 1280,205- codedHeight: 720,206- description: subject.decoders[0].configureCalls[0].description,207- optimizeForLatency: true,208- });209- assert.deepEqual(210- [...new Uint8Array(subject.decoders[0].configureCalls[0].description)],211- [0x62, 0x02, 0x20, 0x00],212- );213- assert.deepEqual(subject.pipeline.format, {214- codec: 'av1', width: 1280, height: 720, epoch: 1,215- });216- assert.deepEqual(subject.formats, [subject.pipeline.format]);217- assert.equal(subject.chunks.length, 1);218- assert.equal(subject.chunks[0].type, 'key');219- assert.equal(subject.chunks[0].timestamp, 3_000);220- assert.deepEqual([...subject.chunks[0].data], [0x32, 0x02, 0xaa, 0xbb]);140+test('unsupported codecs are dropped before decoder construction', () => {141+ for (const codec of ['h265', 'av1']) {142+ const subject = harness();143+ subject.pipeline.processFramePayload({144+ width: 1280,145+ height: 800,146+ codec,147+ keyframe: true,148+ codecConfig: true,149+ sequence: 1,150+ pts_micros: 2_000,151+ discontinuity: false,152+ data: null,153+ }, h264Keyframe());154+155+ assert.equal(subject.decoders.length, 0);156+ assert.equal(subject.chunks.length, 0);157+ assert.deepEqual(subject.pipeline.format, {158+ codec: 'h264', width: 0, height: 0, epoch: 0,159+ });160+ assert.equal(subject.pipeline.snapshot().droppedFrames, 1);161+ }221162 });222163 223164 test('stale decoder callbacks cannot mutate a replacement decoder session', () => {@@ -6,12 +6,12 @@ use tauri::{AppHandle, Emitter};
66 77 use crate::commands::state::AppState;88 9-pub(crate) fn byte_to_codec(value: u8) -> &'static str {9+pub(crate) fn byte_to_codec(value: u8) -> Result<&'static str, String> {1010 match value {11- 1 => "h264",12- 2 => "h265",13- 3 => "av1",14- _ => "h264",11+ 1 => Ok("h264"),12+ unsupported => Err(format!(13+ "Unsupported legacy media codec: {unsupported}; Portal supports H.264 only"14+ )),1515 }1616 }1717 @@ -145,8 +145,6 @@ pub(crate) fn encode_frame_envelope(
145145 })?;146146 let codec = match metadata.codec {147147 "h264" => 1,148- "h265" => 2,149- "av1" => 3,150148 other => return Err(format!("Unsupported frame channel codec: {other}")),151149 };152150 let mut flags = 0_u8;@@ -321,7 +319,7 @@ mod tests {
321319 FrameEnvelopeMetadata {322320 width: 1,323321 height: 1,324- codec: "av1",322+ codec: "h264",325323 keyframe: false,326324 discontinuity: false,327325 codec_config: false,@@ -332,7 +330,7 @@ mod tests {
332330 &[1],333331 )334332 .unwrap();335- assert_eq!(envelope[5], 3);333+ assert_eq!(envelope[5], 1);336334 assert_eq!(&envelope[16..24], &u64::MAX.to_be_bytes());337335 assert_eq!(&envelope[24..32], &u64::MAX.to_be_bytes());338336 assert_eq!(&envelope[32..40], &i64::MIN.to_be_bytes());@@ -351,6 +349,8 @@ mod tests {
351349 capture_timestamp_micros: None,352350 pts_micros: None,353351 };352+ assert!(encode_frame_envelope(metadata("h265"), &[1]).is_err());353+ assert!(encode_frame_envelope(metadata("av1"), &[1]).is_err());354354 assert!(encode_frame_envelope(metadata("vp9"), &[1]).is_err());355355 assert!(encode_frame_envelope(metadata("h264"), &[]).is_err());356356 assert!(@@ -395,6 +395,15 @@ mod tests {
395395 );396396 }397397 398+ 399+ fn legacy_codec_bytes_fail_closed_to_h264_only() {400+ assert_eq!(byte_to_codec(1).unwrap(), "h264");401+ for unsupported in [0, 2, 3, u8::MAX] {402+ let error = byte_to_codec(unsupported).unwrap_err();403+ assert!(error.contains("H.264 only"));404+ }405+ }406+398407 399408 fn frame_channel_slots_are_bounded_and_cannot_underflow() {400409 let in_flight = AtomicUsize::new(0);@@ -444,7 +444,13 @@ pub(crate) async fn run_video_delivery(request: VideoDeliveryRequest) {
444444 }445445 446446 let is_keyframe = header[12] == 1;447- let codec = byte_to_codec(header[13]).to_string();447+ let codec = match byte_to_codec(header[13]) {448+ Ok(codec) => codec.to_string(),449+ Err(error) => {450+ emit_frame_error(&app, media_generation, error);451+ break;452+ }453+ };448454 449455 let mut frame_buf = vec![0u8; frame_len];450456 match tokio::time::timeout(