1import {2 parseAnnexBNals,3 nalsToLengthPrefixed,4 buildAvcDescription,5 avcCodecStr,6 h264NalType,7} from './codecs.js';8import { exactMediaTimestampMicros } from './av-sync.mjs';9import {10 BoundedCadenceWindow,11 BoundedLatencyWindow,12 BoundedValueWindow,13 LatestFramePresenter,14 RollingRateWindow,15} from './stream-metrics.mjs';16import { VideoFormatTransitionGuard } from './video-format-transition.mjs';17import { buildVideoDecoderConfig } from './video-decoder-config.mjs';18import {19 DECODER_RECOVERY_REASONS,20 DecoderRecoveryState,21} from './decoder-recovery.mjs';2223export const MAX_DECODE_QUEUE_SIZE = 2;24const MAX_DECODE_TIMINGS = 8;25const PRESENTER_QUEUE_CAPACITY = 2;2627export function createVideoPipelineSession({28 canvas,29 context,30 requestKeyframe = () => {},31 resetAudioSync = () => {},32 sampleAudioSkew = () => null,33 onFormatChanged = () => {},34 now = () => performance.now(),35 createDecoder = (callbacks) => new VideoDecoder(callbacks),36 createEncodedChunk = (init) => new EncodedVideoChunk(init),37 requestFrame = (callback) => requestAnimationFrame(callback),38 cancelFrame = (handle) => cancelAnimationFrame(handle),39 setTimer = (callback, delayMs) => setTimeout(callback, delayMs),40 cancelTimer = (handle) => clearTimeout(handle),41}) {42 let videoDecoder = null;43 let decoderConfigured = false;44 let activeCodec = 'h264';45 let frameWidth = 0;46 let frameHeight = 0;47 let activeVideoFormatEpoch = 0;48 let droppedFrames = 0;49 let receivedFrames = 0;50 let decoderInputFrames = 0;51 let decoderOutputFrames = 0;52 let presentedFrames = 0;53 let presentationDroppedFrames = 0;54 let lastVideoDrawCompletedAtMs = null;5556 const frontendDeliveryRate = new RollingRateWindow();57 const decoderInputRate = new RollingRateWindow();58 const decoderOutputRate = new RollingRateWindow();59 const presentationRate = new RollingRateWindow();60 const frontendDeliveryCadence = new BoundedCadenceWindow();61 const decoderInputCadence = new BoundedCadenceWindow();62 const decoderOutputCadence = new BoundedCadenceWindow();63 const presentationCadence = new BoundedCadenceWindow();64 const decodeLatency = new BoundedLatencyWindow();65 const clientPresentationLatency = new BoundedLatencyWindow();66 const drawLatency = new BoundedLatencyWindow();67 const avSkew = new BoundedValueWindow();68 const decodeTimings = new Map();69 const decoderRecovery = new DecoderRecoveryState({70 onKeyframeRequest: requestKeyframe,71 });72 const videoFormatTransition = new VideoFormatTransitionGuard();7374 const framePresenter = new LatestFramePresenter({75 requestFrame,76 cancelFrame,77 setTimer,78 cancelTimer,79 now,80 fallbackDelayMs: 25,81 draw: (frame) => {82 const startedAt = now();83 context.drawImage(frame, 0, 0, canvas.width, canvas.height);84 const completedAt = now();85 drawLatency.record(completedAt - startedAt, completedAt);86 lastVideoDrawCompletedAtMs = completedAt;87 },88 onPresent: (metadata, animationFrameTime) => {89 const presentedAt = lastVideoDrawCompletedAtMs ?? animationFrameTime;90 lastVideoDrawCompletedAtMs = null;91 presentedFrames++;92 presentationRate.record(presentedAt);93 presentationCadence.record(presentedAt);94 if (metadata && Number.isFinite(metadata.receivedAt)) {95 clientPresentationLatency.record(presentedAt - metadata.receivedAt, presentedAt);96 }97 if (metadata) {98 const skewMs = sampleAudioSkew(metadata.mediaPtsMicros, presentedAt);99 if (skewMs !== null && skewMs !== undefined) avSkew.record(skewMs, presentedAt);100 }101 },102 onDrop: () => { presentationDroppedFrames++; },103 });104105 function resetAvSync(...args) {106 avSkew.reset();107 resetAudioSync(...args);108 }109110 function teardown() {111 resetAvSync();112 framePresenter.clear();113 decodeTimings.clear();114 if (videoDecoder) {115 try { videoDecoder.close(); } catch (_) {}116 }117 videoDecoder = null;118 decoderConfigured = false;119 }120121 function enterDecoderRecovery(reason, { restart = false, requestKeyframe: shouldRequest = true } = {}) {122 const result = restart123 ? decoderRecovery.restart(reason)124 : decoderRecovery.enter(reason, { requestKeyframe: shouldRequest });125 // Closing WebCodecs and clearing the latest-frame presenter is part of the126 // recovery boundary. Do it immediately rather than allowing already queued127 // frames from a poisoned GOP to continue reaching the canvas.128 teardown();129 return result;130 }131132 function finishVideoChunkEnqueue(keyframe, succeeded) {133 if (succeeded) {134 if (keyframe) decoderRecovery.confirmKeyframeEnqueued(true);135 return;136 }137 // A synchronous decode rejection is a decoder-error episode if playback138 // was previously healthy. If recovery is already active, enter() coalesces139 // with its existing request and leaves the keyframe gate closed.140 enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_ERROR);141 }142143 function reset() {144 teardown();145 decoderRecovery.reset();146 videoFormatTransition.reset();147 activeVideoFormatEpoch = 0;148 droppedFrames = 0;149 receivedFrames = 0;150 decoderInputFrames = 0;151 decoderOutputFrames = 0;152 presentedFrames = 0;153 presentationDroppedFrames = 0;154 frontendDeliveryRate.reset();155 decoderInputRate.reset();156 decoderOutputRate.reset();157 presentationRate.reset();158 frontendDeliveryCadence.reset();159 decoderInputCadence.reset();160 decoderOutputCadence.reset();161 presentationCadence.reset();162 decodeLatency.reset();163 clientPresentationLatency.reset();164 drawLatency.reset();165 }166167 function enqueueVideoChunk(chunk, receivedAt, codecLabel, mediaPtsMicros = null) {168 if (!videoDecoder) return false;169 const enqueuedAt = now();170 while (decodeTimings.size >= MAX_DECODE_TIMINGS) {171 decodeTimings.delete(decodeTimings.keys().next().value);172 }173 decodeTimings.set(chunk.timestamp, { receivedAt, enqueuedAt, mediaPtsMicros });174 try {175 videoDecoder.decode(chunk);176 decoderInputFrames++;177 decoderInputRate.record(enqueuedAt);178 decoderInputCadence.record(enqueuedAt);179 return true;180 } catch (error) {181 decodeTimings.delete(chunk.timestamp);182 droppedFrames++;183 console.warn(`${codecLabel} decode failed:`, error);184 return false;185 }186 }187188 function initWebCodecsDecoder(width, height, desc, codecStr) {189 teardown();190 console.log('WebCodecs configure:', codecStr, 'w:', width, 'h:', height, 'desc:', desc.byteLength, 'bytes');191 const decoder = createDecoder({192 output: (frame) => {193 if (videoDecoder !== decoder) {194 frame.close();195 return;196 }197 const outputAt = now();198 decoderOutputFrames++;199 decoderOutputRate.record(outputAt);200 decoderOutputCadence.record(outputAt);201 const timing = decodeTimings.get(frame.timestamp) ?? null;202 decodeTimings.delete(frame.timestamp);203 if (timing) decodeLatency.record(outputAt - timing.enqueuedAt, outputAt);204 framePresenter.enqueue(frame, timing);205 },206 error: (error) => {207 if (videoDecoder !== decoder) return;208 console.error('VideoDecoder error:', error);209 enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_ERROR);210 },211 });212 videoDecoder = decoder;213 try {214 decoder.configure(buildVideoDecoderConfig({215 codec: codecStr,216 width,217 height,218 description: desc,219 }));220 decoderConfigured = true;221 return true;222 } catch (error) {223 console.warn('WebCodecs configure failed:', error);224 enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_ERROR);225 return false;226 }227 }228229 function commitVideoFrame(plan, keyframe, enqueued) {230 if (!enqueued) {231 finishVideoChunkEnqueue(keyframe, false);232 return false;233 }234 let committed;235 try {236 committed = videoFormatTransition.commit(plan);237 } catch (error) {238 console.warn('discarding stale video format transaction:', error);239 enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET);240 return false;241 }242 if (plan.reconfigure) {243 activeCodec = committed.format.codec;244 frameWidth = committed.format.width;245 frameHeight = committed.format.height;246 activeVideoFormatEpoch = committed.epoch;247 if (canvas.width !== frameWidth) canvas.width = frameWidth;248 if (canvas.height !== frameHeight) canvas.height = frameHeight;249 onFormatChanged({250 codec: activeCodec,251 width: frameWidth,252 height: frameHeight,253 epoch: activeVideoFormatEpoch,254 });255 }256 finishVideoChunkEnqueue(keyframe, true);257 return true;258 }259260 function processFramePayload(payload, binaryData) {261 const receivedAt = now();262 frontendDeliveryRate.record(receivedAt);263 frontendDeliveryCadence.record(receivedAt);264 const {265 width,266 height,267 codec = 'h264',268 keyframe,269 codecConfig = keyframe,270 sequence = null,271 pts_micros: ptsMicros,272 discontinuity = false,273 } = payload;274 if (discontinuity) {275 resetAvSync();276 }277 receivedFrames++;278279 // Binary channels provide an exact view over the IPC ArrayBuffer.280 const raw = binaryData;281 const decoderBacklogged = videoDecoder?.decodeQueueSize >= MAX_DECODE_QUEUE_SIZE;282 if (decoderBacklogged) {283 enterDecoderRecovery(DECODER_RECOVERY_REASONS.FRONTEND_BACKPRESSURE);284 }285 const decoderBoundary = discontinuity || !decoderConfigured;286 let formatPlan;287 try {288 formatPlan = videoFormatTransition.plan({289 sequence,290 codec,291 width,292 height,293 keyframe,294 codecConfig,295 discontinuity: decoderBoundary,296 });297 } catch (error) {298 console.warn('invalid video format transition:', error);299 droppedFrames++;300 enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET);301 return;302 }303 if (formatPlan.action === 'drop-stale') {304 droppedFrames++;305 return;306 }307 if (formatPlan.action === 'recover') {308 droppedFrames++;309 enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET);310 return;311 }312 if (formatPlan.reconfigure) {313 // A delivered discontinuity keyframe is the recovery boundary we were314 // waiting for, not evidence that another keyframe is missing.315 if (discontinuity) {316 enterDecoderRecovery(DECODER_RECOVERY_REASONS.DISCONTINUITY, {317 requestKeyframe: !keyframe,318 });319 } else {320 enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET, {321 requestKeyframe: false,322 });323 }324 }325 if (decoderRecovery.shouldDropFrame({ keyframe })) {326 if (!decoderRecovery.requestIssued) {327 enterDecoderRecovery(DECODER_RECOVERY_REASONS.DECODER_RESET);328 }329 droppedFrames++;330 return;331 }332 const mediaPtsMicros = exactMediaTimestampMicros(ptsMicros);333 const chunkTimestamp = mediaPtsMicros !== null334 ? mediaPtsMicros335 : now() * 1000;336337 const nals = parseAnnexBNals(raw);338 if (keyframe) {339 let sps = null, pps = null;340 for (const nal of nals) {341 const type = h264NalType(nal);342 if (type === 7) sps = nal;343 else if (type === 8) pps = nal;344 }345 if (sps && pps && formatPlan.reconfigure) {346 initWebCodecsDecoder(width, height, buildAvcDescription(sps, pps), avcCodecStr(sps));347 }348 }349 if (!decoderConfigured || !videoDecoder) { droppedFrames++; return; }350 const sliceNals = nals.filter(nal => {351 const type = h264NalType(nal);352 return type !== 7 && type !== 8 && type !== 9;353 });354 if (sliceNals.length > 0) {355 const chunk = createEncodedChunk({356 type: keyframe ? 'key' : 'delta',357 timestamp: chunkTimestamp,358 data: nalsToLengthPrefixed(sliceNals),359 });360 const enqueued = enqueueVideoChunk(chunk, receivedAt, 'h264', mediaPtsMicros);361 commitVideoFrame(formatPlan, keyframe, enqueued);362 }363 }364365 function snapshot(at = now()) {366 return {367 activeCodec,368 frameWidth,369 frameHeight,370 activeVideoFormatEpoch,371 receivedFrames,372 decoderInputFrames,373 decoderOutputFrames,374 presentedFrames,375 droppedFrames,376 presentationDroppedFrames,377 frontendDeliveryFps: frontendDeliveryRate.rate(at),378 decodeFps: decoderInputRate.rate(at),379 decoderOutputFps: decoderOutputRate.rate(at),380 presentFps: presentationRate.rate(at),381 decodePercentiles: decodeLatency.summary(at),382 presentPercentiles: clientPresentationLatency.summary(at),383 drawPercentiles: drawLatency.summary(at),384 deliveryCadence: frontendDeliveryCadence.summary(at),385 decoderInputIntervals: decoderInputCadence.summary(at),386 decoderOutputIntervals: decoderOutputCadence.summary(at),387 presentationIntervals: presentationCadence.summary(at),388 avSkewPercentiles: avSkew.summary(at),389 decoderQueueDepth: videoDecoder?.decodeQueueSize ?? 0,390 decoderQueueCapacity: MAX_DECODE_QUEUE_SIZE,391 presenterQueueDepth: framePresenter.depth,392 presenterQueueCapacity: PRESENTER_QUEUE_CAPACITY,393 recovering: decoderRecovery.recovering,394 };395 }396397 return Object.freeze({398 processFramePayload,399 resetAvSync,400 reset,401 teardown,402 snapshot,403 get format() {404 return {405 codec: activeCodec,406 width: frameWidth,407 height: frameHeight,408 epoch: activeVideoFormatEpoch,409 };410 },411 });412}