- commit
- f1e56973ae130866e280149daed19a941906200a
- parent
- 8d859d2b2c552a39bfd885166e7b1f384f4c16e2
- author
- Dami <dami.etoile@gmail.com>
| M | portal/ | 14 | +- |
| M | portal/ | 552 | ++++-------------------------- |
| A | portal/ | 518 | ++++++++++++++++++++++++++++ |
| A | portal/ | 221 | ++++++++++++ |
@@ -7,19 +7,19 @@ import {
77 DecoderRecoveryState,88 } from './decoder-recovery.mjs';99 10-const main = await readFile(new URL('./main.js', import.meta.url), 'utf8');10+const pipeline = await readFile(new URL('./video-pipeline.mjs', import.meta.url), 'utf8');1111 12-test('connect wiring uses the recovery state instead of removed legacy state', () => {13- assert.doesNotMatch(main, /waitingForDecoderKeyframe/);14- assert.match(main, /decoderRecovery\.reset\(\);/);12+test('video pipeline uses the recovery state instead of removed legacy state', () => {13+ assert.doesNotMatch(pipeline, /waitingForDecoderKeyframe/);14+ assert.match(pipeline, /decoderRecovery\.reset\(\);/);1515 });1616 1717 test('decoder callbacks cannot mutate a replacement decoder session', () => {1818 assert.match(19- main,19+ pipeline,2020 /output: \(frame\) => \{\s*if \(videoDecoder !== decoder\) \{\s*frame\.close\(\);\s*return;/,2121 );22- assert.match(main, /error: \(e\) => \{\s*if \(videoDecoder !== decoder\) return;/);22+ assert.match(pipeline, /error: \(error\) => \{\s*if \(videoDecoder !== decoder\) return;/);2323 });2424 2525 test('a delivered discontinuity keyframe resets without requesting its replacement', () => {@@ -37,7 +37,7 @@ test('a delivered discontinuity keyframe resets without requesting its replaceme
3737 assert.equal(recovery.confirmKeyframeEnqueued(true), true);3838 assert.deepEqual(requests, []);3939 assert.match(40- main,40+ pipeline,4141 /enterDecoderRecovery\(DECODER_RECOVERY_REASONS\.DISCONTINUITY, \{\s*requestKeyframe: !keyframe,/,4242 );4343 });@@ -1,16 +1,3 @@
1-import {2- parseAnnexBNals,3- nalsToLengthPrefixed,4- buildAvcDescription,5- avcCodecStr,6- h264NalType,7- buildHvcDescription,8- hevcCodecStr,9- hevcNalType,10- parseAv1Obus,11- av1CodecStr,12- buildAv1Description,13-} from './codecs.js';141 import {152 HeldInputState,163 MAX_HELD_KEYS,@@ -63,13 +50,6 @@ import {
6350 } from './audio-envelope.mjs';6451 import { BoundedAudioMessageTracker } from './audio-ring.mjs';6552 import { audioButtonPresentation } from './audio-ui.mjs';66-import {67- BoundedCadenceWindow,68- BoundedLatencyWindow,69- BoundedValueWindow,70- LatestFramePresenter,71- RollingRateWindow,72-} from './stream-metrics.mjs';7353 import {7454 ControllerActivationGate,7555 ControllerActionRepeater,@@ -90,12 +70,9 @@ import {
9070 fitStreamSurface,9171 streamAspectKey,9272 } from './window-geometry.mjs';93-import { VideoFormatTransitionGuard } from './video-format-transition.mjs';94-import { buildVideoDecoderConfig } from './video-decoder-config.mjs';9573 import {96- DECODER_RECOVERY_REASONS,97- DecoderRecoveryState,98-} from './decoder-recovery.mjs';74+ createVideoPipelineSession,75+} from './video-pipeline.mjs';9976 import {10077 AdaptiveFeedbackPublisher,10178 formatAdaptiveDecision,@@ -126,8 +103,6 @@ let inputCapabilities = {
126103 gamepad: false,127104 control: false,128105 };129-let frameWidth = 0;130-let frameHeight = 0;131106 let fittedStreamAspect = null;132107 let pendingStreamFit = null;133108 let lastObservedWindowSize = null;@@ -138,7 +113,6 @@ let remotePointerVisible = false;
138113 let activeFrameChannel = null;139114 let activeFrameGeneration = null;140115 let activeFrameSession = null;141-let activeVideoFormatEpoch = 0;142116 let activeAudioChannel = null;143117 let activePointerChannel = null;144118 let activePointerSession = null;@@ -545,7 +519,7 @@ function resetAudioTelemetry() {
545519 audioSilentDurationMicros = 0;546520 audioTransportDropped = 0;547521 audioFrontendDropped = 0;548- resetAvSyncTelemetry(activeAudioSession);522+ videoPipeline.resetAvSync(activeAudioSession);549523 updateAudioUI();550524 }551525 @@ -716,7 +690,7 @@ async function initializeAudioPipeline(session) {
716690 717691 async function teardownAudioPipeline(resetStatus = true) {718692 const session = activeAudioSession;719- resetAvSyncTelemetry(session);693+ videoPipeline.resetAvSync(session);720694 activeAudioSession = null;721695 activeAudioChannel = null;722696 audioAvailable = false;@@ -995,7 +969,7 @@ async function disconnect() {
995969 activePointerChannel = null;996970 activePointerSession = null;997971 await teardownAudioPipeline();998- teardownDecoderPipeline();972+ videoPipeline.teardown();999973 connected = false;1000974 controlMode = false;1001975 inputCapabilities = {@@ -1069,16 +1043,12 @@ function togglePanel(force) {
10691043 10701044 // ─── WebCodecs detection ──────────────────────────────────────────────────────10711045 let hasWebCodecs = ('VideoDecoder' in window);1072-let videoDecoder = null;1073-let decoderConfigured = false;10741046 10751047 (async function detectWebCodecs() {10761048 await invoke('set_webcodecs_available', { available: hasWebCodecs });10771049 })();10781050 10791051 // ─── Frame decoding ───────────────────────────────────────────────────────────1080-let activeCodec = 'h264';1081-let droppedFrames = 0;10821052 let transportDroppedFrames = 0;10831053 let transportObjectDroppedFrames = null;10841054 let transportLateObjectDroppedFrames = null;@@ -1091,32 +1061,12 @@ let transportIntervalStats = null;
10911061 let frontendIpcSendDurationStats = null;10921062 let rustTimingWindow = null;10931063 let networkDiagnostics = null;1094-let receivedFrames = 0;1095-let decoderInputFrames = 0;1096-let decoderOutputFrames = 0;1097-let presentedFrames = 0;1098-let presentationDroppedFrames = 0;10991064 let lastTransportFps = 0;11001065 let lastFrontendSendFps = 0;1101-const frontendDeliveryRate = new RollingRateWindow();1102-const decoderInputRate = new RollingRateWindow();1103-const decoderOutputRate = new RollingRateWindow();1104-const presentationRate = new RollingRateWindow();1105-const frontendDeliveryCadence = new BoundedCadenceWindow();1106-const decoderInputCadence = new BoundedCadenceWindow();1107-const decoderOutputCadence = new BoundedCadenceWindow();1108-const presentationCadence = new BoundedCadenceWindow();1109-const decodeLatency = new BoundedLatencyWindow();1110-const clientPresentationLatency = new BoundedLatencyWindow();1111-const drawLatency = new BoundedLatencyWindow();1112-const avSkew = new BoundedValueWindow();1113-const decodeTimings = new Map();1114-const MAX_DECODE_TIMINGS = 8;11151066 let streamPathMode = 'unknown';11161067 let streamTransportMode = 'unknown';11171068 let streamRttMs = null;11181069 let lastMediaSequence = null;1119-const MAX_DECODE_QUEUE_SIZE = 2;11201070 11211071 function requestDecoderKeyframe(reason) {11221072 if (!Number.isSafeInteger(activeFrameGeneration) || activeFrameGeneration < 1) return;@@ -1128,35 +1078,7 @@ function requestDecoderKeyframe(reason) {
11281078 });11291079 }11301080 1131-const decoderRecovery = new DecoderRecoveryState({1132- onKeyframeRequest: requestDecoderKeyframe,1133-});1134-const videoFormatTransition = new VideoFormatTransitionGuard();1135-1136-function enterDecoderRecovery(reason, { restart = false, requestKeyframe = true } = {}) {1137- const result = restart1138- ? decoderRecovery.restart(reason)1139- : decoderRecovery.enter(reason, { requestKeyframe });1140- // Closing WebCodecs and clearing the latest-frame presenter is part of the1141- // recovery boundary. Do it immediately rather than allowing already queued1142- // frames from a poisoned GOP to continue reaching the canvas.1143- teardownDecoderPipeline();1144- return result;1145-}1146-1147-function finishVideoChunkEnqueue(keyframe, succeeded) {1148- if (succeeded) {1149- if (keyframe) decoderRecovery.confirmKeyframeEnqueued(true);1150- return;1151- }1152- // A synchronous decode rejection is a decoder-error episode if playback1153- // was previously healthy. If recovery is already active, enter() coalesces1154- // with its existing request and leaves the keyframe gate closed.1155- enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_ERROR);1156-}1157-1158-function resetAvSyncTelemetry(session = activeAudioSession, { recoverRing = false } = {}) {1159- avSkew.reset();1081+function resetAudioSyncTelemetry(session = activeAudioSession, { recoverRing = false } = {}) {11601082 if (!session) return;11611083 session.audioTimeline = null;11621084 session.avSyncEpoch = nextAvSyncEpoch(session.avSyncEpoch);@@ -1178,72 +1100,40 @@ function sampleAvSkew(videoPtsMicros, presentationTimeMs) {
11781100 || !audioAvailable11791101 || audioContext?.state !== 'running'11801102 || typeof audioContext.getOutputTimestamp !== 'function'1181- ) return;1103+ ) return null;11821104 11831105 let outputTimestamp;11841106 try {11851107 outputTimestamp = audioContext.getOutputTimestamp();11861108 } catch (_) {1187- return;1109+ return null;11881110 }11891111 const audioPtsMicros = projectAudioMediaPtsMicros(11901112 session.audioTimeline,11911113 outputTimestamp,11921114 presentationTimeMs,11931115 );1194- const skewMs = audioMinusVideoSkewMs(audioPtsMicros, videoPtsMicros);1195- if (skewMs !== null) avSkew.record(skewMs, presentationTimeMs);1116+ return audioMinusVideoSkewMs(audioPtsMicros, videoPtsMicros);11961117 }11971118 11981119 const canvas = document.getElementById('frame-canvas');11991120 const remotePointer = document.getElementById('remote-pointer');12001121 const ctx = canvas.getContext('2d');1201-let lastVideoDrawCompletedAtMs = null;1202-const framePresenter = new LatestFramePresenter({1203- requestFrame: (callback) => requestAnimationFrame(callback),1204- cancelFrame: (handle) => cancelAnimationFrame(handle),1205- setTimer: (callback, delayMs) => setTimeout(callback, delayMs),1206- cancelTimer: (handle) => clearTimeout(handle),1207- now: () => performance.now(),1208- fallbackDelayMs: 25,1209- draw: (frame) => {1210- const startedAt = performance.now();1211- ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);1212- const completedAt = performance.now();1213- drawLatency.record(completedAt - startedAt, completedAt);1214- lastVideoDrawCompletedAtMs = completedAt;1215- },1216- onPresent: (metadata, animationFrameTime) => {1217- const now = lastVideoDrawCompletedAtMs ?? animationFrameTime;1218- lastVideoDrawCompletedAtMs = null;1219- presentedFrames++;1220- presentationRate.record(now);1221- presentationCadence.record(now);1222- if (metadata && Number.isFinite(metadata.receivedAt)) {1223- clientPresentationLatency.record(now - metadata.receivedAt, now);1224- }1225- if (metadata) sampleAvSkew(metadata.mediaPtsMicros, now);1122+const videoPipeline = createVideoPipelineSession({1123+ hasWebCodecs,1124+ canvas,1125+ context: ctx,1126+ requestKeyframe: requestDecoderKeyframe,1127+ resetAudioSync: resetAudioSyncTelemetry,1128+ sampleAudioSkew: sampleAvSkew,1129+ onFormatChanged: () => {1130+ sizeCanvasToIncomingStream();1131+ if (connected) void fitWindowToIncomingStream();12261132 },1227- onDrop: () => { presentationDroppedFrames++; },12281133 });12291134 1230-function teardownDecoderPipeline() {1231- resetAvSyncTelemetry();1232- framePresenter.clear();1233- decodeTimings.clear();1234- if (videoDecoder) {1235- try { videoDecoder.close(); } catch (_) {}1236- }1237- videoDecoder = null;1238- decoderConfigured = false;1239-}1240-12411135 function resetStreamTelemetry() {1242- teardownDecoderPipeline();1243- decoderRecovery.reset();1244- videoFormatTransition.reset();1245- activeVideoFormatEpoch = 0;1246- droppedFrames = 0;1136+ videoPipeline.reset();12471137 transportDroppedFrames = 0;12481138 transportObjectDroppedFrames = null;12491139 transportLateObjectDroppedFrames = null;@@ -1256,24 +1146,8 @@ function resetStreamTelemetry() {
12561146 frontendIpcSendDurationStats = null;12571147 rustTimingWindow = null;12581148 networkDiagnostics = null;1259- receivedFrames = 0;1260- decoderInputFrames = 0;1261- decoderOutputFrames = 0;1262- presentedFrames = 0;1263- presentationDroppedFrames = 0;12641149 lastTransportFps = 0;12651150 lastFrontendSendFps = 0;1266- frontendDeliveryRate.reset();1267- decoderInputRate.reset();1268- decoderOutputRate.reset();1269- presentationRate.reset();1270- frontendDeliveryCadence.reset();1271- decoderInputCadence.reset();1272- decoderOutputCadence.reset();1273- presentationCadence.reset();1274- decodeLatency.reset();1275- clientPresentationLatency.reset();1276- drawLatency.reset();12771151 streamPathMode = 'unknown';12781152 streamTransportMode = 'unknown';12791153 streamRttMs = null;@@ -1281,68 +1155,6 @@ function resetStreamTelemetry() {
12811155 adaptiveDecision = null;12821156 }12831157 1284-function enqueueVideoChunk(chunk, receivedAt, codecLabel, mediaPtsMicros = null) {1285- if (!videoDecoder) return false;1286- const enqueuedAt = performance.now();1287- while (decodeTimings.size >= MAX_DECODE_TIMINGS) {1288- decodeTimings.delete(decodeTimings.keys().next().value);1289- }1290- decodeTimings.set(chunk.timestamp, { receivedAt, enqueuedAt, mediaPtsMicros });1291- try {1292- videoDecoder.decode(chunk);1293- decoderInputFrames++;1294- decoderInputRate.record(enqueuedAt);1295- decoderInputCadence.record(enqueuedAt);1296- return true;1297- } catch (error) {1298- decodeTimings.delete(chunk.timestamp);1299- droppedFrames++;1300- console.warn(`${codecLabel} decode failed:`, error);1301- return false;1302- }1303-}1304-1305-function initWebCodecsDecoder(width, height, desc, codecStr) {1306- teardownDecoderPipeline();1307- console.log('WebCodecs configure:', codecStr, 'w:', width, 'h:', height, 'desc:', desc.byteLength, 'bytes');1308- const decoder = new VideoDecoder({1309- output: (frame) => {1310- if (videoDecoder !== decoder) {1311- frame.close();1312- return;1313- }1314- const now = performance.now();1315- decoderOutputFrames++;1316- decoderOutputRate.record(now);1317- decoderOutputCadence.record(now);1318- const timing = decodeTimings.get(frame.timestamp) ?? null;1319- decodeTimings.delete(frame.timestamp);1320- if (timing) decodeLatency.record(now - timing.enqueuedAt, now);1321- framePresenter.enqueue(frame, timing);1322- },1323- error: (e) => {1324- if (videoDecoder !== decoder) return;1325- console.error('VideoDecoder error:', e);1326- enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_ERROR);1327- }1328- });1329- videoDecoder = decoder;1330- try {1331- decoder.configure(buildVideoDecoderConfig({1332- codec: codecStr,1333- width,1334- height,1335- description: desc,1336- }));1337- decoderConfigured = true;1338- return true;1339- } catch (error) {1340- console.warn('WebCodecs configure failed:', error);1341- enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_ERROR);1342- return false;1343- }1344-}1345-13461158 function formatCadence(summary) {13471159 if (!summary.count) return '—';13481160 return `${summary.p50.toFixed(1)} / ${summary.p95.toFixed(1)} / ${summary.p99.toFixed(1)} / ${summary.max.toFixed(1)} ms · >25/>33 ${summary.over25Ms}/${summary.over33Ms}`;@@ -1356,30 +1168,19 @@ function formatRustDurationSummary(summary) {
13561168 13571169 function updateStreamStats() {13581170 const now = performance.now();1359- const frontendDeliveryFps = frontendDeliveryRate.rate(now);1360- const decodeFps = decoderInputRate.rate(now);1361- const decoderOutputFps = decoderOutputRate.rate(now);1362- const presentFps = presentationRate.rate(now);1363- const decodePercentiles = decodeLatency.summary(now);1364- const presentPercentiles = clientPresentationLatency.summary(now);1365- const drawPercentiles = drawLatency.summary(now);1366- const deliveryCadence = frontendDeliveryCadence.summary(now);1367- const decoderInputIntervals = decoderInputCadence.summary(now);1368- const decoderOutputIntervals = decoderOutputCadence.summary(now);1369- const presentationIntervals = presentationCadence.summary(now);1370- const avSkewPercentiles = avSkew.summary(now);1171+ const video = videoPipeline.snapshot(now);13711172 const networkPresentation = networkDiagnosticsPresentation(networkDiagnostics);1372- document.getElementById('stream-received').textContent = receivedFrames;1373- document.getElementById('stream-decoder-input').textContent = decoderInputFrames;1374- document.getElementById('stream-decoder-output').textContent = decoderOutputFrames;1375- document.getElementById('stream-presented').textContent = presentedFrames;1376- document.getElementById('stream-decode-queue').textContent = videoDecoder?.decodeQueueSize ?? 0;1377- document.getElementById('stream-present-queue').textContent = framePresenter.depth;1173+ document.getElementById('stream-received').textContent = video.receivedFrames;1174+ document.getElementById('stream-decoder-input').textContent = video.decoderInputFrames;1175+ document.getElementById('stream-decoder-output').textContent = video.decoderOutputFrames;1176+ document.getElementById('stream-presented').textContent = video.presentedFrames;1177+ document.getElementById('stream-decode-queue').textContent = video.decoderQueueDepth;1178+ document.getElementById('stream-present-queue').textContent = video.presenterQueueDepth;13781179 const discardTelemetry = formatVideoDiscardTelemetry({13791180 transportDroppedFrames,13801181 frontendDroppedFrames,1381- decoderDroppedFrames: droppedFrames,1382- presenterOverwrittenFrames: presentationDroppedFrames,1182+ decoderDroppedFrames: video.droppedFrames,1183+ presenterOverwrittenFrames: video.presentationDroppedFrames,13831184 });13841185 document.getElementById('stream-dropped').textContent = discardTelemetry.total;13851186 document.getElementById('stream-transport-dropped').textContent = discardTelemetry.transport;@@ -1408,27 +1209,27 @@ function updateStreamStats() {
14081209 document.getElementById('stream-present-dropped').textContent = discardTelemetry.presenterOverwrite;14091210 document.getElementById('stream-transport-fps').textContent = lastTransportFps.toFixed(1);14101211 document.getElementById('stream-ipc-send-fps').textContent = lastFrontendSendFps.toFixed(1);1411- document.getElementById('stream-frontend-fps').textContent = frontendDeliveryFps.toFixed(1);1412- document.getElementById('stream-decode-fps').textContent = decodeFps.toFixed(1);1413- document.getElementById('stream-decoder-output-fps').textContent = decoderOutputFps.toFixed(1);1414- document.getElementById('stream-present-fps').textContent = presentFps.toFixed(1);1415- document.getElementById('stream-av-skew').textContent = avSkewPercentiles.count1416- ? `${formatSignedMilliseconds(avSkewPercentiles.p50)} / ${formatSignedMilliseconds(avSkewPercentiles.p95)} / ${avSkewPercentiles.maxAbsolute.toFixed(1)} ms`1212+ document.getElementById('stream-frontend-fps').textContent = video.frontendDeliveryFps.toFixed(1);1213+ document.getElementById('stream-decode-fps').textContent = video.decodeFps.toFixed(1);1214+ document.getElementById('stream-decoder-output-fps').textContent = video.decoderOutputFps.toFixed(1);1215+ document.getElementById('stream-present-fps').textContent = video.presentFps.toFixed(1);1216+ document.getElementById('stream-av-skew').textContent = video.avSkewPercentiles.count1217+ ? `${formatSignedMilliseconds(video.avSkewPercentiles.p50)} / ${formatSignedMilliseconds(video.avSkewPercentiles.p95)} / ${video.avSkewPercentiles.maxAbsolute.toFixed(1)} ms`14171218 : '—';1418- document.getElementById('stream-decode-latency').textContent = decodePercentiles.count1419- ? `${decodePercentiles.p50.toFixed(1)} / ${decodePercentiles.p95.toFixed(1)} ms`1219+ document.getElementById('stream-decode-latency').textContent = video.decodePercentiles.count1220+ ? `${video.decodePercentiles.p50.toFixed(1)} / ${video.decodePercentiles.p95.toFixed(1)} ms`14201221 : '—';1421- document.getElementById('stream-present-latency').textContent = presentPercentiles.count1422- ? `${presentPercentiles.p50.toFixed(1)} / ${presentPercentiles.p95.toFixed(1)} ms`1222+ document.getElementById('stream-present-latency').textContent = video.presentPercentiles.count1223+ ? `${video.presentPercentiles.p50.toFixed(1)} / ${video.presentPercentiles.p95.toFixed(1)} ms`14231224 : '—';1424- document.getElementById('stream-draw-latency').textContent = drawPercentiles.count1425- ? `${drawPercentiles.p50.toFixed(2)} / ${drawPercentiles.p95.toFixed(2)} / ${drawPercentiles.p99.toFixed(2)} / ${drawPercentiles.max.toFixed(2)} ms`1225+ document.getElementById('stream-draw-latency').textContent = video.drawPercentiles.count1226+ ? `${video.drawPercentiles.p50.toFixed(2)} / ${video.drawPercentiles.p95.toFixed(2)} / ${video.drawPercentiles.p99.toFixed(2)} / ${video.drawPercentiles.max.toFixed(2)} ms`14261227 : '—';1427- document.getElementById('stream-delivery-cadence').textContent = formatCadence(deliveryCadence);1428- document.getElementById('stream-decoder-input-cadence').textContent = formatCadence(decoderInputIntervals);1429- document.getElementById('stream-decoder-output-cadence').textContent = formatCadence(decoderOutputIntervals);1430- document.getElementById('stream-present-cadence').textContent = formatCadence(presentationIntervals);1431- document.getElementById('stream-codec').textContent = activeCodec;1228+ document.getElementById('stream-delivery-cadence').textContent = formatCadence(video.deliveryCadence);1229+ document.getElementById('stream-decoder-input-cadence').textContent = formatCadence(video.decoderInputIntervals);1230+ document.getElementById('stream-decoder-output-cadence').textContent = formatCadence(video.decoderOutputIntervals);1231+ document.getElementById('stream-present-cadence').textContent = formatCadence(video.presentationIntervals);1232+ document.getElementById('stream-codec').textContent = video.activeCodec;14321233 document.getElementById('stream-transport').textContent = streamTransportMode;14331234 document.getElementById('stream-path').textContent = streamPathMode;14341235 document.getElementById('stream-rtt').textContent = Number.isFinite(streamRttMs)@@ -1451,20 +1252,20 @@ function updateStreamStats() {
14511252 lastSequence: lastMediaSequence,14521253 frontendQueueDepth: frontendQueueStats?.depth ?? 0,14531254 frontendQueueCapacity: frontendQueueStats?.capacity ?? 4,1454- decoderQueueDepth: videoDecoder?.decodeQueueSize ?? 0,1455- decoderQueueCapacity: MAX_DECODE_QUEUE_SIZE,1456- presenterQueueDepth: framePresenter.depth,1457- presenterQueueCapacity: 2,1255+ decoderQueueDepth: video.decoderQueueDepth,1256+ decoderQueueCapacity: video.decoderQueueCapacity,1257+ presenterQueueDepth: video.presenterQueueDepth,1258+ presenterQueueCapacity: video.presenterQueueCapacity,14581259 transportDroppedTotal: transportDroppedFrames,14591260 frontendDroppedTotal: frontendDroppedFrames,1460- decoderDroppedTotal: droppedFrames,1461- presenterDroppedTotal: presentationDroppedFrames,1261+ decoderDroppedTotal: video.droppedFrames,1262+ presenterDroppedTotal: video.presentationDroppedFrames,14621263 // Portal has no clock-synchronized capture-to-delivery measurement yet.14631264 // IPC send duration is a different boundary and must not be mislabeled.14641265 transportDeliveryP95Ms: null,1465- decodeLatencyP95Ms: decodePercentiles.p95,1466- presentationLatencyP95Ms: presentPercentiles.p95,1467- resyncActive: decoderRecovery.recovering || frontendResyncStats?.active === true,1266+ decodeLatencyP95Ms: video.decodePercentiles.p95,1267+ presentationLatencyP95Ms: video.presentPercentiles.p95,1268+ resyncActive: video.recovering || frontendResyncStats?.active === true,14681269 });14691270 }14701271 @@ -1479,6 +1280,7 @@ function clientChromeHeight() {
14791280 }14801281 14811282 function sizeCanvasToIncomingStream() {1283+ const { width: frameWidth, height: frameHeight } = videoPipeline.format;14821284 if (frameWidth < 1 || frameHeight < 1) return;14831285 const main = canvas.parentElement;14841286 if (!main) return;@@ -1510,6 +1312,11 @@ async function applyClientWindowGeometry(geometry, unmaximize) {
15101312 }15111313 15121314 async function fitWindowToIncomingStream() {1315+ const {1316+ width: frameWidth,1317+ height: frameHeight,1318+ epoch: activeVideoFormatEpoch,1319+ } = videoPipeline.format;15131320 if (!connected || frameWidth < 1 || frameHeight < 1) return false;15141321 const aspect = streamAspectKey(frameWidth, frameHeight);15151322 if (fittedStreamAspect === aspect || pendingStreamFit?.aspect === aspect) return false;@@ -1523,13 +1330,14 @@ async function fitWindowToIncomingStream() {
15231330 availableHeight: window.screen.availHeight,15241331 });15251332 const applied = await applyClientWindowGeometry(geometry, true);1526- const currentAspect = frameWidth > 0 && frameHeight > 01527- ? streamAspectKey(frameWidth, frameHeight)1333+ const currentFormat = videoPipeline.format;1334+ const currentAspect = currentFormat.width > 0 && currentFormat.height > 01335+ ? streamAspectKey(currentFormat.width, currentFormat.height)15281336 : null;15291337 const current = pendingStreamFit === request15301338 && connected15311339 && activeFrameGeneration === request.generation1532- && activeVideoFormatEpoch === request.epoch1340+ && currentFormat.epoch === request.epoch15331341 && currentAspect === request.aspect;15341342 if (pendingStreamFit === request) pendingStreamFit = null;15351343 if (applied && current) fittedStreamAspect = request.aspect;@@ -1543,6 +1351,7 @@ function scheduleStreamWindowAspectCorrection() {
15431351 if (streamWindowResizeTimer !== null) clearTimeout(streamWindowResizeTimer);15441352 streamWindowResizeTimer = setTimeout(() => {15451353 streamWindowResizeTimer = null;1354+ const { width: frameWidth, height: frameHeight } = videoPipeline.format;15461355 if (!connected || frameWidth < 1 || frameHeight < 1) return;15471356 const current = { width: window.innerWidth, height: window.innerHeight };15481357 let geometry;@@ -1564,224 +1373,6 @@ function scheduleStreamWindowAspectCorrection() {
15641373 }, 80);15651374 }15661375 1567-function commitVideoFrame(plan, keyframe, enqueued) {1568- if (!enqueued) {1569- finishVideoChunkEnqueue(keyframe, false);1570- return false;1571- }1572- let committed;1573- try {1574- committed = videoFormatTransition.commit(plan);1575- } catch (error) {1576- console.warn('discarding stale video format transaction:', error);1577- enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET);1578- return false;1579- }1580- if (plan.reconfigure) {1581- activeCodec = committed.format.codec;1582- frameWidth = committed.format.width;1583- frameHeight = committed.format.height;1584- activeVideoFormatEpoch = committed.epoch;1585- if (canvas.width !== frameWidth) canvas.width = frameWidth;1586- if (canvas.height !== frameHeight) canvas.height = frameHeight;1587- sizeCanvasToIncomingStream();1588- if (connected) void fitWindowToIncomingStream();1589- }1590- finishVideoChunkEnqueue(keyframe, true);1591- return true;1592-}1593-1594-function processFramePayload(payload, binaryData = null) {1595- const receivedAt = performance.now();1596- frontendDeliveryRate.record(receivedAt);1597- frontendDeliveryCadence.record(receivedAt);1598- const {1599- width,1600- height,1601- data,1602- codec = 'h264',1603- keyframe,1604- codecConfig = keyframe,1605- sequence = null,1606- pts_micros: ptsMicros,1607- discontinuity = false,1608- } = payload;1609- if (discontinuity) resetAvSyncTelemetry();1610- receivedFrames++;1611-1612- if (hasWebCodecs) {1613- // Binary channels provide an exact view over the IPC ArrayBuffer. The1614- // base64 branch only supports an older in-process sender during migration.1615- const raw = binaryData ?? Uint8Array.from(atob(data), c => c.charCodeAt(0));1616- const decoderBacklogged = videoDecoder?.decodeQueueSize >= MAX_DECODE_QUEUE_SIZE;1617- if (decoderBacklogged) {1618- enterDecoderRecovery(DECODER_RECOVERY_REASONS.FRONTEND_BACKPRESSURE);1619- }1620- const decoderBoundary = discontinuity || !decoderConfigured;1621- let formatPlan;1622- try {1623- formatPlan = videoFormatTransition.plan({1624- sequence,1625- codec,1626- width,1627- height,1628- keyframe,1629- codecConfig,1630- discontinuity: decoderBoundary,1631- });1632- } catch (error) {1633- console.warn('invalid video format transition:', error);1634- droppedFrames++;1635- enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET);1636- return;1637- }1638- if (formatPlan.action === 'drop-stale') {1639- droppedFrames++;1640- return;1641- }1642- if (formatPlan.action === 'recover') {1643- droppedFrames++;1644- enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET);1645- return;1646- }1647- if (formatPlan.reconfigure) {1648- // A delivered discontinuity keyframe is the recovery boundary we were1649- // waiting for, not evidence that another keyframe is missing.1650- if (discontinuity) {1651- enterDecoderRecovery(DECODER_RECOVERY_REASONS.DISCONTINUITY, {1652- requestKeyframe: !keyframe,1653- });1654- } else {1655- enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET, {1656- requestKeyframe: false,1657- });1658- }1659- }1660- if (decoderRecovery.shouldDropFrame({ keyframe })) {1661- if (!decoderRecovery.requestIssued) {1662- enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET);1663- }1664- droppedFrames++;1665- return;1666- }1667- const mediaPtsMicros = exactMediaTimestampMicros(ptsMicros);1668- const chunkTimestamp = mediaPtsMicros !== null1669- ? mediaPtsMicros1670- : performance.now() * 1000;1671-1672- if (codec === 'av1') {1673- // ── AV1 path ──1674- const obus = parseAv1Obus(raw);1675- if (keyframe) {1676- const seqHeader = obus.find(o => o.type === 12);1677- if (seqHeader && formatPlan.reconfigure) {1678- initWebCodecsDecoder(width, height, buildAv1Description(seqHeader.data), av1CodecStr(seqHeader.data));1679- }1680- }1681- if (!decoderConfigured || !videoDecoder) { droppedFrames++; return; }1682- const frameObus = obus.filter(o => o.type !== 2 && o.type !== 12);1683- if (frameObus.length > 0) {1684- let totalLen = 0;1685- for (const o of frameObus) totalLen += o.data.length;1686- const chunkData = new Uint8Array(totalLen);1687- let off = 0;1688- for (const o of frameObus) { chunkData.set(o.data, off); off += o.data.length; }1689- const chunk = new EncodedVideoChunk({1690- type: keyframe ? 'key' : 'delta',1691- timestamp: chunkTimestamp,1692- data: chunkData,1693- });1694- const enqueued = enqueueVideoChunk(chunk, receivedAt, 'av1', mediaPtsMicros);1695- commitVideoFrame(formatPlan, keyframe, enqueued);1696- }1697-1698- } else if (codec === 'h265') {1699- // ── H.265 path ──1700- const nals = parseAnnexBNals(raw);1701- if (keyframe) {1702- let vps = null, sps = null, pps = null;1703- for (const nal of nals) {1704- const t = hevcNalType(nal);1705- if (t === 32) vps = nal;1706- else if (t === 33) sps = nal;1707- else if (t === 34) pps = nal;1708- }1709- if (vps && sps && pps && formatPlan.reconfigure) {1710- initWebCodecsDecoder(width, height, buildHvcDescription(vps, sps, pps), hevcCodecStr(sps));1711- }1712- }1713- if (!decoderConfigured || !videoDecoder) { droppedFrames++; return; }1714- const sliceNals = nals.filter(nal => {1715- const t = hevcNalType(nal);1716- return t !== 32 && t !== 33 && t !== 34 && t !== 35;1717- });1718- if (sliceNals.length > 0) {1719- const chunk = new EncodedVideoChunk({1720- type: keyframe ? 'key' : 'delta',1721- timestamp: chunkTimestamp,1722- data: nalsToLengthPrefixed(sliceNals),1723- });1724- const enqueued = enqueueVideoChunk(chunk, receivedAt, 'hevc', mediaPtsMicros);1725- commitVideoFrame(formatPlan, keyframe, enqueued);1726- }1727-1728- } else {1729- // ── H.264 path (default) ──1730- const nals = parseAnnexBNals(raw);1731- if (keyframe) {1732- let sps = null, pps = null;1733- for (const nal of nals) {1734- const t = h264NalType(nal);1735- if (t === 7) sps = nal;1736- else if (t === 8) pps = nal;1737- }1738- if (sps && pps && formatPlan.reconfigure) {1739- initWebCodecsDecoder(width, height, buildAvcDescription(sps, pps), avcCodecStr(sps));1740- }1741- }1742- if (!decoderConfigured || !videoDecoder) { droppedFrames++; return; }1743- const sliceNals = nals.filter(nal => {1744- const t = h264NalType(nal);1745- return t !== 7 && t !== 8 && t !== 9;1746- });1747- if (sliceNals.length > 0) {1748- const chunk = new EncodedVideoChunk({1749- type: keyframe ? 'key' : 'delta',1750- timestamp: chunkTimestamp,1751- data: nalsToLengthPrefixed(sliceNals),1752- });1753- const enqueued = enqueueVideoChunk(chunk, receivedAt, 'h264', mediaPtsMicros);1754- commitVideoFrame(formatPlan, keyframe, enqueued);1755- }1756- }1757- } else {1758- // JPEG fallback (openh264 decode → JPEG encode in Rust)1759- const dimensionsChanged = frameWidth !== width || frameHeight !== height;1760- frameWidth = width;1761- frameHeight = height;1762- if (canvas.width !== width) canvas.width = width;1763- if (canvas.height !== height) canvas.height = height;1764- if (dimensionsChanged) sizeCanvasToIncomingStream();1765- if (dimensionsChanged && connected) void fitWindowToIncomingStream();1766- const bytes = Uint8Array.from(atob(data), c => c.charCodeAt(0));1767- const blob = new Blob([bytes], { type: 'image/jpeg' });1768- const url = URL.createObjectURL(blob);1769- const img = new Image();1770- img.onload = () => {1771- const drawStartedAt = performance.now();1772- ctx.drawImage(img, 0, 0, canvas.width, canvas.height);1773- URL.revokeObjectURL(url);1774- const now = performance.now();1775- presentedFrames++;1776- presentationRate.record(now);1777- presentationCadence.record(now);1778- drawLatency.record(now - drawStartedAt, now);1779- clientPresentationLatency.record(now - receivedAt, now);1780- };1781- img.src = url;1782- }1783-}1784-17851376 function sendFrameAcknowledgment(generation) {17861377 void invoke('iroh_client_ack_frame', { generation }).catch((error) => {17871378 console.warn('frame acknowledgment failed:', error);@@ -1808,7 +1399,7 @@ function activateConnectedFrameSession(session, generation) {
18081399 if (pendingError) throw new Error(pendingError.error || 'Media connection failed');18091400 for (const payload of activation.legacyFrames) {18101401 if (isCurrentFrameGeneration(payload?.generation, generation)) {1811- processFramePayload(payload);1402+ videoPipeline.processFramePayload(payload);18121403 }18131404 }18141405 }@@ -1832,7 +1423,7 @@ function handleBinaryFrameMessage(message, session) {
18321423 // discard. WebCodecs remains independently bounded by decodeQueueSize.18331424 acknowledgeFrame(session);18341425 acknowledged = true;1835- processFramePayload({1426+ videoPipeline.processFramePayload({18361427 width: frame.width,18371428 height: frame.height,18381429 data: null,@@ -1878,7 +1469,7 @@ function requestNativeAudioStop(session) {
18781469 18791470 function disableAudioForSession(detail, session = activeAudioSession) {18801471 if (!session || session !== activeAudioSession) return;1881- resetAvSyncTelemetry(session);1472+ videoPipeline.resetAvSync(session);18821473 session.failed = true;18831474 session.failureDetail = detail;18841475 audioAvailable = false;@@ -1919,7 +1510,7 @@ function handleBinaryAudioMessage(message, session) {
19191510 }19201511 const queuedAudioPackets = audioDecoder.decodeQueueSize;19211512 if (packet.discontinuity || queuedAudioPackets >= MAX_AUDIO_DECODE_QUEUE_SIZE) {1922- resetAvSyncTelemetry(session, { recoverRing: true });1513+ videoPipeline.resetAvSync(session, { recoverRing: true });19231514 audioDecoderDropped += queuedAudioPackets;19241515 configureAudioDecoder(session);19251516 }@@ -1946,7 +1537,7 @@ listen('frame', (event) => {
19461537 for (const generation of delivery.acknowledgments) {19471538 sendFrameAcknowledgment(generation);19481539 }1949- if (delivery.accepted) processFramePayload(event.payload);1540+ if (delivery.accepted) videoPipeline.processFramePayload(event.payload);19501541 } catch (error) {19511542 failFrameSession(session, error);19521543 }@@ -2250,6 +1841,7 @@ async function releaseNativeCursorGrab(releaseGeneration) {
22501841 }22511842 22521843 function currentPointerSurfaceSize() {1844+ const { width: frameWidth, height: frameHeight } = videoPipeline.format;22531845 return resolvePointerSurfaceSize(22541846 pointerSurfaceDimensions,22551847 frameWidth,@@ -0,0 +1,518 @@
1+import {2+ parseAnnexBNals,3+ nalsToLengthPrefixed,4+ buildAvcDescription,5+ avcCodecStr,6+ h264NalType,7+ buildHvcDescription,8+ hevcCodecStr,9+ hevcNalType,10+ parseAv1Obus,11+ av1CodecStr,12+ buildAv1Description,13+} from './codecs.js';14+import { exactMediaTimestampMicros } from './av-sync.mjs';15+import {16+ BoundedCadenceWindow,17+ BoundedLatencyWindow,18+ BoundedValueWindow,19+ LatestFramePresenter,20+ RollingRateWindow,21+} from './stream-metrics.mjs';22+import { VideoFormatTransitionGuard } from './video-format-transition.mjs';23+import { buildVideoDecoderConfig } from './video-decoder-config.mjs';24+import {25+ DECODER_RECOVERY_REASONS,26+ DecoderRecoveryState,27+} from './decoder-recovery.mjs';28+29+export const MAX_DECODE_QUEUE_SIZE = 2;30+const MAX_DECODE_TIMINGS = 8;31+const PRESENTER_QUEUE_CAPACITY = 2;32+33+export function createVideoPipelineSession({34+ hasWebCodecs,35+ canvas,36+ context,37+ requestKeyframe = () => {},38+ resetAudioSync = () => {},39+ sampleAudioSkew = () => null,40+ onFormatChanged = () => {},41+ now = () => performance.now(),42+ createDecoder = (callbacks) => new VideoDecoder(callbacks),43+ createEncodedChunk = (init) => new EncodedVideoChunk(init),44+ decodeBase64 = (data) => Uint8Array.from(atob(data), c => c.charCodeAt(0)),45+ createBlob = (parts, options) => new Blob(parts, options),46+ createObjectUrl = (blob) => URL.createObjectURL(blob),47+ revokeObjectUrl = (url) => URL.revokeObjectURL(url),48+ createImage = () => new Image(),49+ requestFrame = (callback) => requestAnimationFrame(callback),50+ cancelFrame = (handle) => cancelAnimationFrame(handle),51+ setTimer = (callback, delayMs) => setTimeout(callback, delayMs),52+ cancelTimer = (handle) => clearTimeout(handle),53+}) {54+ let videoDecoder = null;55+ let decoderConfigured = false;56+ let activeCodec = 'h264';57+ let frameWidth = 0;58+ let frameHeight = 0;59+ let activeVideoFormatEpoch = 0;60+ let droppedFrames = 0;61+ let receivedFrames = 0;62+ let decoderInputFrames = 0;63+ let decoderOutputFrames = 0;64+ let presentedFrames = 0;65+ let presentationDroppedFrames = 0;66+ let lastVideoDrawCompletedAtMs = null;67+68+ const frontendDeliveryRate = new RollingRateWindow();69+ const decoderInputRate = new RollingRateWindow();70+ const decoderOutputRate = new RollingRateWindow();71+ const presentationRate = new RollingRateWindow();72+ const frontendDeliveryCadence = new BoundedCadenceWindow();73+ const decoderInputCadence = new BoundedCadenceWindow();74+ const decoderOutputCadence = new BoundedCadenceWindow();75+ const presentationCadence = new BoundedCadenceWindow();76+ const decodeLatency = new BoundedLatencyWindow();77+ const clientPresentationLatency = new BoundedLatencyWindow();78+ const drawLatency = new BoundedLatencyWindow();79+ const avSkew = new BoundedValueWindow();80+ const decodeTimings = new Map();81+ const decoderRecovery = new DecoderRecoveryState({82+ onKeyframeRequest: requestKeyframe,83+ });84+ const videoFormatTransition = new VideoFormatTransitionGuard();85+86+ const framePresenter = new LatestFramePresenter({87+ requestFrame,88+ cancelFrame,89+ setTimer,90+ cancelTimer,91+ now,92+ fallbackDelayMs: 25,93+ draw: (frame) => {94+ const startedAt = now();95+ context.drawImage(frame, 0, 0, canvas.width, canvas.height);96+ const completedAt = now();97+ drawLatency.record(completedAt - startedAt, completedAt);98+ lastVideoDrawCompletedAtMs = completedAt;99+ },100+ onPresent: (metadata, animationFrameTime) => {101+ const presentedAt = lastVideoDrawCompletedAtMs ?? animationFrameTime;102+ lastVideoDrawCompletedAtMs = null;103+ presentedFrames++;104+ presentationRate.record(presentedAt);105+ presentationCadence.record(presentedAt);106+ if (metadata && Number.isFinite(metadata.receivedAt)) {107+ clientPresentationLatency.record(presentedAt - metadata.receivedAt, presentedAt);108+ }109+ if (metadata) {110+ const skewMs = sampleAudioSkew(metadata.mediaPtsMicros, presentedAt);111+ if (skewMs !== null && skewMs !== undefined) avSkew.record(skewMs, presentedAt);112+ }113+ },114+ onDrop: () => { presentationDroppedFrames++; },115+ });116+117+ function resetAvSync(...args) {118+ avSkew.reset();119+ resetAudioSync(...args);120+ }121+122+ function teardown() {123+ resetAvSync();124+ framePresenter.clear();125+ decodeTimings.clear();126+ if (videoDecoder) {127+ try { videoDecoder.close(); } catch (_) {}128+ }129+ videoDecoder = null;130+ decoderConfigured = false;131+ }132+133+ function enterDecoderRecovery(reason, { restart = false, requestKeyframe: shouldRequest = true } = {}) {134+ const result = restart135+ ? decoderRecovery.restart(reason)136+ : decoderRecovery.enter(reason, { requestKeyframe: shouldRequest });137+ // Closing WebCodecs and clearing the latest-frame presenter is part of the138+ // recovery boundary. Do it immediately rather than allowing already queued139+ // frames from a poisoned GOP to continue reaching the canvas.140+ teardown();141+ return result;142+ }143+144+ function finishVideoChunkEnqueue(keyframe, succeeded) {145+ if (succeeded) {146+ if (keyframe) decoderRecovery.confirmKeyframeEnqueued(true);147+ return;148+ }149+ // A synchronous decode rejection is a decoder-error episode if playback150+ // was previously healthy. If recovery is already active, enter() coalesces151+ // with its existing request and leaves the keyframe gate closed.152+ enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_ERROR);153+ }154+155+ function reset() {156+ teardown();157+ decoderRecovery.reset();158+ videoFormatTransition.reset();159+ activeVideoFormatEpoch = 0;160+ droppedFrames = 0;161+ receivedFrames = 0;162+ decoderInputFrames = 0;163+ decoderOutputFrames = 0;164+ presentedFrames = 0;165+ presentationDroppedFrames = 0;166+ frontendDeliveryRate.reset();167+ decoderInputRate.reset();168+ decoderOutputRate.reset();169+ presentationRate.reset();170+ frontendDeliveryCadence.reset();171+ decoderInputCadence.reset();172+ decoderOutputCadence.reset();173+ presentationCadence.reset();174+ decodeLatency.reset();175+ clientPresentationLatency.reset();176+ drawLatency.reset();177+ }178+179+ function enqueueVideoChunk(chunk, receivedAt, codecLabel, mediaPtsMicros = null) {180+ if (!videoDecoder) return false;181+ const enqueuedAt = now();182+ while (decodeTimings.size >= MAX_DECODE_TIMINGS) {183+ decodeTimings.delete(decodeTimings.keys().next().value);184+ }185+ decodeTimings.set(chunk.timestamp, { receivedAt, enqueuedAt, mediaPtsMicros });186+ try {187+ videoDecoder.decode(chunk);188+ decoderInputFrames++;189+ decoderInputRate.record(enqueuedAt);190+ decoderInputCadence.record(enqueuedAt);191+ return true;192+ } catch (error) {193+ decodeTimings.delete(chunk.timestamp);194+ droppedFrames++;195+ console.warn(`${codecLabel} decode failed:`, error);196+ return false;197+ }198+ }199+200+ function initWebCodecsDecoder(width, height, desc, codecStr) {201+ teardown();202+ console.log('WebCodecs configure:', codecStr, 'w:', width, 'h:', height, 'desc:', desc.byteLength, 'bytes');203+ const decoder = createDecoder({204+ output: (frame) => {205+ if (videoDecoder !== decoder) {206+ frame.close();207+ return;208+ }209+ const outputAt = now();210+ decoderOutputFrames++;211+ decoderOutputRate.record(outputAt);212+ decoderOutputCadence.record(outputAt);213+ const timing = decodeTimings.get(frame.timestamp) ?? null;214+ decodeTimings.delete(frame.timestamp);215+ if (timing) decodeLatency.record(outputAt - timing.enqueuedAt, outputAt);216+ framePresenter.enqueue(frame, timing);217+ },218+ error: (error) => {219+ if (videoDecoder !== decoder) return;220+ console.error('VideoDecoder error:', error);221+ enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_ERROR);222+ },223+ });224+ videoDecoder = decoder;225+ try {226+ decoder.configure(buildVideoDecoderConfig({227+ codec: codecStr,228+ width,229+ height,230+ description: desc,231+ }));232+ decoderConfigured = true;233+ return true;234+ } catch (error) {235+ console.warn('WebCodecs configure failed:', error);236+ enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_ERROR);237+ return false;238+ }239+ }240+241+ function commitVideoFrame(plan, keyframe, enqueued) {242+ if (!enqueued) {243+ finishVideoChunkEnqueue(keyframe, false);244+ return false;245+ }246+ let committed;247+ try {248+ committed = videoFormatTransition.commit(plan);249+ } catch (error) {250+ console.warn('discarding stale video format transaction:', error);251+ enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET);252+ return false;253+ }254+ if (plan.reconfigure) {255+ activeCodec = committed.format.codec;256+ frameWidth = committed.format.width;257+ frameHeight = committed.format.height;258+ activeVideoFormatEpoch = committed.epoch;259+ if (canvas.width !== frameWidth) canvas.width = frameWidth;260+ if (canvas.height !== frameHeight) canvas.height = frameHeight;261+ onFormatChanged({262+ codec: activeCodec,263+ width: frameWidth,264+ height: frameHeight,265+ epoch: activeVideoFormatEpoch,266+ });267+ }268+ finishVideoChunkEnqueue(keyframe, true);269+ return true;270+ }271+272+ function processFramePayload(payload, binaryData = null) {273+ const receivedAt = now();274+ frontendDeliveryRate.record(receivedAt);275+ frontendDeliveryCadence.record(receivedAt);276+ const {277+ width,278+ height,279+ data,280+ codec = 'h264',281+ keyframe,282+ codecConfig = keyframe,283+ sequence = null,284+ pts_micros: ptsMicros,285+ discontinuity = false,286+ } = payload;287+ if (discontinuity) {288+ resetAvSync();289+ }290+ receivedFrames++;291+292+ if (hasWebCodecs) {293+ // Binary channels provide an exact view over the IPC ArrayBuffer. The294+ // base64 branch only supports an older in-process sender during migration.295+ const raw = binaryData ?? decodeBase64(data);296+ const decoderBacklogged = videoDecoder?.decodeQueueSize >= MAX_DECODE_QUEUE_SIZE;297+ if (decoderBacklogged) {298+ enterDecoderRecovery(DECODER_RECOVERY_REASONS.FRONTEND_BACKPRESSURE);299+ }300+ const decoderBoundary = discontinuity || !decoderConfigured;301+ let formatPlan;302+ try {303+ formatPlan = videoFormatTransition.plan({304+ sequence,305+ codec,306+ width,307+ height,308+ keyframe,309+ codecConfig,310+ discontinuity: decoderBoundary,311+ });312+ } catch (error) {313+ console.warn('invalid video format transition:', error);314+ droppedFrames++;315+ enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET);316+ return;317+ }318+ if (formatPlan.action === 'drop-stale') {319+ droppedFrames++;320+ return;321+ }322+ if (formatPlan.action === 'recover') {323+ droppedFrames++;324+ enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET);325+ return;326+ }327+ if (formatPlan.reconfigure) {328+ // A delivered discontinuity keyframe is the recovery boundary we were329+ // waiting for, not evidence that another keyframe is missing.330+ if (discontinuity) {331+ enterDecoderRecovery(DECODER_RECOVERY_REASONS.DISCONTINUITY, {332+ requestKeyframe: !keyframe,333+ });334+ } else {335+ enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET, {336+ requestKeyframe: false,337+ });338+ }339+ }340+ if (decoderRecovery.shouldDropFrame({ keyframe })) {341+ if (!decoderRecovery.requestIssued) {342+ enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET);343+ }344+ droppedFrames++;345+ return;346+ }347+ const mediaPtsMicros = exactMediaTimestampMicros(ptsMicros);348+ const chunkTimestamp = mediaPtsMicros !== null349+ ? mediaPtsMicros350+ : now() * 1000;351+352+ if (codec === 'av1') {353+ // ── AV1 path ──354+ const obus = parseAv1Obus(raw);355+ if (keyframe) {356+ const seqHeader = obus.find(o => o.type === 12);357+ if (seqHeader && formatPlan.reconfigure) {358+ initWebCodecsDecoder(width, height, buildAv1Description(seqHeader.data), av1CodecStr(seqHeader.data));359+ }360+ }361+ if (!decoderConfigured || !videoDecoder) { droppedFrames++; return; }362+ const frameObus = obus.filter(o => o.type !== 2 && o.type !== 12);363+ if (frameObus.length > 0) {364+ let totalLen = 0;365+ for (const o of frameObus) totalLen += o.data.length;366+ const chunkData = new Uint8Array(totalLen);367+ let off = 0;368+ for (const o of frameObus) { chunkData.set(o.data, off); off += o.data.length; }369+ const chunk = createEncodedChunk({370+ type: keyframe ? 'key' : 'delta',371+ timestamp: chunkTimestamp,372+ data: chunkData,373+ });374+ const enqueued = enqueueVideoChunk(chunk, receivedAt, 'av1', mediaPtsMicros);375+ commitVideoFrame(formatPlan, keyframe, enqueued);376+ }377+378+ } else if (codec === 'h265') {379+ // ── H.265 path ──380+ const nals = parseAnnexBNals(raw);381+ if (keyframe) {382+ let vps = null, sps = null, pps = null;383+ for (const nal of nals) {384+ const type = hevcNalType(nal);385+ if (type === 32) vps = nal;386+ else if (type === 33) sps = nal;387+ else if (type === 34) pps = nal;388+ }389+ if (vps && sps && pps && formatPlan.reconfigure) {390+ initWebCodecsDecoder(width, height, buildHvcDescription(vps, sps, pps), hevcCodecStr(sps));391+ }392+ }393+ if (!decoderConfigured || !videoDecoder) { droppedFrames++; return; }394+ const sliceNals = nals.filter(nal => {395+ const type = hevcNalType(nal);396+ return type !== 32 && type !== 33 && type !== 34 && type !== 35;397+ });398+ if (sliceNals.length > 0) {399+ const chunk = createEncodedChunk({400+ type: keyframe ? 'key' : 'delta',401+ timestamp: chunkTimestamp,402+ data: nalsToLengthPrefixed(sliceNals),403+ });404+ const enqueued = enqueueVideoChunk(chunk, receivedAt, 'hevc', mediaPtsMicros);405+ commitVideoFrame(formatPlan, keyframe, enqueued);406+ }407+408+ } else {409+ // ── H.264 path (default) ──410+ const nals = parseAnnexBNals(raw);411+ if (keyframe) {412+ let sps = null, pps = null;413+ for (const nal of nals) {414+ const type = h264NalType(nal);415+ if (type === 7) sps = nal;416+ else if (type === 8) pps = nal;417+ }418+ if (sps && pps && formatPlan.reconfigure) {419+ initWebCodecsDecoder(width, height, buildAvcDescription(sps, pps), avcCodecStr(sps));420+ }421+ }422+ if (!decoderConfigured || !videoDecoder) { droppedFrames++; return; }423+ const sliceNals = nals.filter(nal => {424+ const type = h264NalType(nal);425+ return type !== 7 && type !== 8 && type !== 9;426+ });427+ if (sliceNals.length > 0) {428+ const chunk = createEncodedChunk({429+ type: keyframe ? 'key' : 'delta',430+ timestamp: chunkTimestamp,431+ data: nalsToLengthPrefixed(sliceNals),432+ });433+ const enqueued = enqueueVideoChunk(chunk, receivedAt, 'h264', mediaPtsMicros);434+ commitVideoFrame(formatPlan, keyframe, enqueued);435+ }436+ }437+ } else {438+ // JPEG fallback (openh264 decode → JPEG encode in Rust)439+ const dimensionsChanged = frameWidth !== width || frameHeight !== height;440+ frameWidth = width;441+ frameHeight = height;442+ if (canvas.width !== width) canvas.width = width;443+ if (canvas.height !== height) canvas.height = height;444+ if (dimensionsChanged) {445+ onFormatChanged({446+ codec: activeCodec,447+ width: frameWidth,448+ height: frameHeight,449+ epoch: activeVideoFormatEpoch,450+ });451+ }452+ const bytes = decodeBase64(data);453+ const blob = createBlob([bytes], { type: 'image/jpeg' });454+ const url = createObjectUrl(blob);455+ const image = createImage();456+ image.onload = () => {457+ const drawStartedAt = now();458+ context.drawImage(image, 0, 0, canvas.width, canvas.height);459+ revokeObjectUrl(url);460+ const presentedAt = now();461+ presentedFrames++;462+ presentationRate.record(presentedAt);463+ presentationCadence.record(presentedAt);464+ drawLatency.record(presentedAt - drawStartedAt, presentedAt);465+ clientPresentationLatency.record(presentedAt - receivedAt, presentedAt);466+ };467+ image.src = url;468+ }469+ }470+471+ function snapshot(at = now()) {472+ return {473+ activeCodec,474+ frameWidth,475+ frameHeight,476+ activeVideoFormatEpoch,477+ receivedFrames,478+ decoderInputFrames,479+ decoderOutputFrames,480+ presentedFrames,481+ droppedFrames,482+ presentationDroppedFrames,483+ frontendDeliveryFps: frontendDeliveryRate.rate(at),484+ decodeFps: decoderInputRate.rate(at),485+ decoderOutputFps: decoderOutputRate.rate(at),486+ presentFps: presentationRate.rate(at),487+ decodePercentiles: decodeLatency.summary(at),488+ presentPercentiles: clientPresentationLatency.summary(at),489+ drawPercentiles: drawLatency.summary(at),490+ deliveryCadence: frontendDeliveryCadence.summary(at),491+ decoderInputIntervals: decoderInputCadence.summary(at),492+ decoderOutputIntervals: decoderOutputCadence.summary(at),493+ presentationIntervals: presentationCadence.summary(at),494+ avSkewPercentiles: avSkew.summary(at),495+ decoderQueueDepth: videoDecoder?.decodeQueueSize ?? 0,496+ decoderQueueCapacity: MAX_DECODE_QUEUE_SIZE,497+ presenterQueueDepth: framePresenter.depth,498+ presenterQueueCapacity: PRESENTER_QUEUE_CAPACITY,499+ recovering: decoderRecovery.recovering,500+ };501+ }502+503+ return Object.freeze({504+ processFramePayload,505+ resetAvSync,506+ reset,507+ teardown,508+ snapshot,509+ get format() {510+ return {511+ codec: activeCodec,512+ width: frameWidth,513+ height: frameHeight,514+ epoch: activeVideoFormatEpoch,515+ };516+ },517+ });518+}@@ -0,0 +1,221 @@
1+import assert from 'node:assert/strict';2+import test from 'node:test';3+4+import {5+ MAX_DECODE_QUEUE_SIZE,6+ createVideoPipelineSession,7+} from './video-pipeline.mjs';8+9+const h264Keyframe = () => new Uint8Array([10+ 0, 0, 0, 1, 0x67, 0x64, 0x00, 0x1f,11+ 0, 0, 0, 1, 0x68, 0xee, 0x3c, 0x80,12+ 0, 0, 0, 1, 0x09, 0xf0,13+ 0, 0, 0, 1, 0x65, 0x88, 0x84,14+]);15+16+const h264Delta = () => new Uint8Array([17+ 0, 0, 0, 1, 0x09, 0xf0,18+ 0, 0, 0, 1, 0x41, 0x9a,19+]);20+21+function harness({ hasWebCodecs = true } = {}) {22+ let clock = 10;23+ const decoders = [];24+ const chunks = [];25+ const keyframeRequests = [];26+ const audioResets = [];27+ const formats = [];28+ const draws = [];29+ const revokedUrls = [];30+ const images = [];31+ const canvas = { width: 0, height: 0 };32+ const context = {33+ drawImage: (...args) => draws.push(args),34+ };35+36+ class FakeDecoder {37+ constructor(callbacks) {38+ this.callbacks = callbacks;39+ this.decodeQueueSize = 0;40+ this.decoded = [];41+ this.configureCalls = [];42+ this.closeCalls = 0;43+ decoders.push(this);44+ }45+46+ configure(config) { this.configureCalls.push(config); }47+ decode(chunk) { this.decoded.push(chunk); }48+ close() { this.closeCalls++; }49+ }50+51+ const pipeline = createVideoPipelineSession({52+ hasWebCodecs,53+ canvas,54+ context,55+ requestKeyframe: (reason) => keyframeRequests.push(reason),56+ resetAudioSync: (...args) => audioResets.push(args),57+ sampleAudioSkew: () => null,58+ onFormatChanged: (format) => formats.push(format),59+ now: () => clock,60+ createDecoder: (callbacks) => new FakeDecoder(callbacks),61+ createEncodedChunk: (init) => {62+ const chunk = { ...init };63+ chunks.push(chunk);64+ return chunk;65+ },66+ decodeBase64: () => new Uint8Array([1, 2, 3]),67+ createBlob: (parts, options) => ({ parts, options }),68+ createObjectUrl: () => 'blob:test-frame',69+ revokeObjectUrl: (url) => revokedUrls.push(url),70+ createImage: () => {71+ const image = { onload: null, src: null };72+ images.push(image);73+ return image;74+ },75+ requestFrame: () => 1,76+ cancelFrame: () => {},77+ setTimer: () => 2,78+ cancelTimer: () => {},79+ });80+81+ return {82+ pipeline,83+ canvas,84+ decoders,85+ chunks,86+ keyframeRequests,87+ audioResets,88+ formats,89+ draws,90+ revokedUrls,91+ images,92+ advance(milliseconds = 1) { clock += milliseconds; },93+ };94+}95+96+function processH264Keyframe(subject, overrides = {}) {97+ subject.pipeline.processFramePayload({98+ width: 1280,99+ height: 800,100+ codec: 'h264',101+ keyframe: true,102+ codecConfig: true,103+ sequence: 1,104+ pts_micros: 1_000,105+ discontinuity: false,106+ data: null,107+ ...overrides,108+ }, h264Keyframe());109+}110+111+test('configured H.264 keyframes commit format only after decode enqueue', () => {112+ const subject = harness();113+ processH264Keyframe(subject);114+115+ assert.equal(subject.decoders.length, 1);116+ assert.equal(subject.decoders[0].configureCalls.length, 1);117+ assert.deepEqual(subject.decoders[0].configureCalls[0], {118+ codec: 'avc1.64001f',119+ codedWidth: 1280,120+ codedHeight: 800,121+ description: subject.decoders[0].configureCalls[0].description,122+ optimizeForLatency: true,123+ });124+ assert.deepEqual(subject.pipeline.format, {125+ codec: 'h264', width: 1280, height: 800, epoch: 1,126+ });127+ assert.deepEqual(subject.formats, [subject.pipeline.format]);128+ assert.equal(subject.chunks.length, 1);129+ assert.equal(subject.chunks[0].type, 'key');130+ assert.deepEqual([...subject.chunks[0].data], [0, 0, 0, 3, 0x65, 0x88, 0x84]);131+ assert.deepEqual(subject.keyframeRequests, []);132+ const stats = subject.pipeline.snapshot();133+ assert.equal(stats.receivedFrames, 1);134+ assert.equal(stats.decoderInputFrames, 1);135+ assert.equal(stats.droppedFrames, 0);136+ assert.equal(stats.recovering, false);137+ assert.equal(stats.decoderQueueCapacity, MAX_DECODE_QUEUE_SIZE);138+});139+140+test('stale decoder callbacks cannot mutate a replacement decoder session', () => {141+ const subject = harness();142+ processH264Keyframe(subject);143+ const stale = subject.decoders[0];144+145+ processH264Keyframe(subject, { sequence: 2, discontinuity: true, pts_micros: 2_000 });146+ assert.equal(subject.decoders.length, 2);147+ const staleFrame = { timestamp: 1_000, closeCalls: 0, close() { this.closeCalls++; } };148+ stale.callbacks.output(staleFrame);149+ stale.callbacks.error(new Error('stale decoder error'));150+151+ assert.equal(staleFrame.closeCalls, 1);152+ assert.equal(subject.pipeline.snapshot().decoderOutputFrames, 0);153+ assert.deepEqual(subject.keyframeRequests, []);154+ assert.equal(subject.pipeline.format.epoch, 2);155+});156+157+test('decode queue overload enters one bounded recovery episode and drops deltas', () => {158+ const subject = harness();159+ processH264Keyframe(subject);160+ subject.decoders[0].decodeQueueSize = MAX_DECODE_QUEUE_SIZE;161+ subject.pipeline.processFramePayload({162+ width: 1280,163+ height: 800,164+ codec: 'h264',165+ keyframe: false,166+ codecConfig: false,167+ sequence: 2,168+ pts_micros: 2_000,169+ data: null,170+ }, h264Delta());171+172+ assert.deepEqual(subject.keyframeRequests, ['frontend-backpressure']);173+ assert.equal(subject.decoders[0].closeCalls, 1);174+ const stats = subject.pipeline.snapshot();175+ assert.equal(stats.receivedFrames, 2);176+ assert.equal(stats.decoderInputFrames, 1);177+ assert.equal(stats.droppedFrames, 1);178+ assert.equal(stats.recovering, true);179+ assert.equal(stats.decoderQueueDepth, 0);180+});181+182+test('JPEG fallback keeps its asynchronous base64 draw behavior', () => {183+ const subject = harness({ hasWebCodecs: false });184+ subject.pipeline.processFramePayload({185+ width: 640,186+ height: 400,187+ data: 'AQID',188+ keyframe: true,189+ }, new Uint8Array([9, 9, 9]));190+191+ assert.deepEqual(subject.pipeline.format, {192+ codec: 'h264', width: 640, height: 400, epoch: 0,193+ });194+ assert.deepEqual(subject.formats, [subject.pipeline.format]);195+ assert.equal(subject.canvas.width, 640);196+ assert.equal(subject.canvas.height, 400);197+ assert.equal(subject.images[0].src, 'blob:test-frame');198+ assert.deepEqual(subject.draws, []);199+ assert.deepEqual(subject.revokedUrls, []);200+201+ subject.advance();202+ subject.images[0].onload();203+ assert.equal(subject.draws.length, 1);204+ assert.deepEqual(subject.revokedUrls, ['blob:test-frame']);205+ assert.equal(subject.pipeline.snapshot().presentedFrames, 1);206+});207+208+test('reset closes decoding state but preserves the last dimensions and codec quirk', () => {209+ const subject = harness();210+ processH264Keyframe(subject);211+ subject.pipeline.reset();212+213+ assert.equal(subject.decoders[0].closeCalls, 1);214+ assert.deepEqual(subject.pipeline.format, {215+ codec: 'h264', width: 1280, height: 800, epoch: 0,216+ });217+ const stats = subject.pipeline.snapshot();218+ assert.equal(stats.receivedFrames, 0);219+ assert.equal(stats.decoderInputFrames, 0);220+ assert.equal(stats.recovering, true);221+});