- commit
- 5c7f4372067b3f99dc38c46563b07a37a02caa01
- parent
- 8e8749fe97c262665e0b603432e706a26c04424f
- author
- Dami <dami.etoile@gmail.com>
| M | portal/ | 14 | +- |
| M | portal/ | 158 | +++++++++++++++++++----------- |
| M | portal/ | 122 | +++++++++++++++++++----- |
@@ -336,7 +336,7 @@ test('adaptive feedback preserves bounded queue fallbacks and pressure signals',
336336 });337337 });338338 339-test('adaptive feedback projects the JPEG one-slot queue and overwrite counter', () => {339+test('adaptive feedback projects the bounded JPEG decode-plus-latest queue and overwrite counter', () => {340340 const { runtime, events } = subject();341341 runtime.prepareConnection();342342 const session = runtime.openFrameSession();@@ -344,21 +344,21 @@ test('adaptive feedback projects the JPEG one-slot queue and overwrite counter',
344344 events.length = 0;345345 346346 assert.equal(runtime.publishAdaptiveFeedback(videoSnapshot({347- presenterQueueDepth: 1,348- presenterQueueCapacity: 1,347+ presenterQueueDepth: 2,348+ presenterQueueCapacity: 2,349349 presentationDroppedFrames: 3,350350 })), true);351- assert.equal(events[0].snapshot.presenterQueueDepth, 1);352- assert.equal(events[0].snapshot.presenterQueueCapacity, 1);351+ assert.equal(events[0].snapshot.presenterQueueDepth, 2);352+ assert.equal(events[0].snapshot.presenterQueueCapacity, 2);353353 assert.equal(events[0].snapshot.presenterDroppedTotal, 3);354354 355355 events.length = 0;356356 assert.equal(runtime.publishAdaptiveFeedback(videoSnapshot({357357 presenterQueueDepth: 0,358- presenterQueueCapacity: 1,358+ presenterQueueCapacity: 2,359359 presentationDroppedFrames: 3,360360 })), true);361361 assert.equal(events[0].snapshot.presenterQueueDepth, 0);362- assert.equal(events[0].snapshot.presenterQueueCapacity, 1);362+ assert.equal(events[0].snapshot.presenterQueueCapacity, 2);363363 assert.equal(events[0].snapshot.presenterDroppedTotal, 3);364364 });@@ -23,7 +23,7 @@ import {
2323 export const MAX_DECODE_QUEUE_SIZE = 2;2424 const MAX_DECODE_TIMINGS = 8;2525 const PRESENTER_QUEUE_CAPACITY = 2;26-const JPEG_PRESENTER_QUEUE_CAPACITY = 1;26+const JPEG_PRESENTER_QUEUE_CAPACITY = 2;2727 2828 export function createVideoPipelineSession({2929 hasWebCodecs,@@ -41,6 +41,7 @@ export function createVideoPipelineSession({
4141 createObjectUrl = (blob) => URL.createObjectURL(blob),4242 revokeObjectUrl = (url) => URL.revokeObjectURL(url),4343 createImage = () => new Image(),44+ abortImage = (image) => image.removeAttribute('src'),4445 requestFrame = (callback) => requestAnimationFrame(callback),4546 cancelFrame = (handle) => cancelAnimationFrame(handle),4647 setTimer = (callback, delayMs) => setTimeout(callback, delayMs),@@ -60,7 +61,8 @@ export function createVideoPipelineSession({
6061 let presentationDroppedFrames = 0;6162 let lastVideoDrawCompletedAtMs = null;6263 let jpegFrameSequence = 0;63- let pendingJpegFrame = null;64+ let activeJpegDecode = null;65+ let queuedJpegFrame = null;6466 6567 const frontendDeliveryRate = new RollingRateWindow();6668 const decoderInputRate = new RollingRateWindow();@@ -116,29 +118,107 @@ export function createVideoPipelineSession({
116118 resetAudioSync(...args);117119 }118120 119- function releaseJpegFrame(frame) {121+ function releaseActiveJpegDecode(frame, { abort = false } = {}) {120122 if (!frame || frame.released) return;121123 frame.released = true;122124 frame.image.onload = null;123125 frame.image.onerror = null;126+ if (abort) {127+ try {128+ abortImage(frame.image);129+ } catch (error) {130+ console.warn('could not abort stale JPEG decode:', error);131+ }132+ }124133 revokeObjectUrl(frame.url);125134 }126135 127- function discardPendingJpegFrame({ overwritten = false } = {}) {128- const frame = pendingJpegFrame;129- if (!frame) return;130- pendingJpegFrame = null;131- if (overwritten) presentationDroppedFrames++;132- releaseJpegFrame(frame);136+ function startQueuedJpegDecode() {137+ if (activeJpegDecode || !queuedJpegFrame) return;138+ const pending = queuedJpegFrame;139+ queuedJpegFrame = null;140+141+ let url = null;142+ let image;143+ try {144+ url = createObjectUrl(pending.blob);145+ image = createImage();146+ } catch (error) {147+ if (url !== null) revokeObjectUrl(url);148+ droppedFrames++;149+ console.warn('could not start JPEG decode:', error);150+ startQueuedJpegDecode();151+ return;152+ }153+154+ const frame = {155+ ...pending,156+ image,157+ url,158+ released: false,159+ superseded: false,160+ };161+ activeJpegDecode = frame;162+ image.onload = () => {163+ if (activeJpegDecode !== frame) return;164+ activeJpegDecode = null;165+ try {166+ if (!frame.superseded && queuedJpegFrame === null) {167+ const drawStartedAt = now();168+ context.drawImage(image, 0, 0, canvas.width, canvas.height);169+ const presentedAt = now();170+ presentedFrames++;171+ presentationRate.record(presentedAt);172+ presentationCadence.record(presentedAt);173+ drawLatency.record(presentedAt - drawStartedAt, presentedAt);174+ clientPresentationLatency.record(175+ presentedAt - frame.receivedAt,176+ presentedAt,177+ );178+ }179+ } finally {180+ releaseActiveJpegDecode(frame);181+ startQueuedJpegDecode();182+ }183+ };184+ image.onerror = () => {185+ if (activeJpegDecode !== frame) return;186+ activeJpegDecode = null;187+ if (!frame.superseded && queuedJpegFrame === null) droppedFrames++;188+ releaseActiveJpegDecode(frame);189+ startQueuedJpegDecode();190+ };191+ try {192+ image.src = url;193+ } catch (error) {194+ if (activeJpegDecode === frame) activeJpegDecode = null;195+ droppedFrames++;196+ releaseActiveJpegDecode(frame, { abort: true });197+ console.warn('could not assign JPEG decode source:', error);198+ startQueuedJpegDecode();199+ }133200 }134201 135- function isCurrentJpegFrame(frame) {136- return pendingJpegFrame === frame && frame.sequence === jpegFrameSequence;202+ function enqueueJpegFrame(frame) {203+ if (!activeJpegDecode) {204+ queuedJpegFrame = frame;205+ startQueuedJpegDecode();206+ return;207+ }208+ if (!activeJpegDecode.superseded) {209+ activeJpegDecode.superseded = true;210+ presentationDroppedFrames++;211+ }212+ if (queuedJpegFrame) presentationDroppedFrames++;213+ queuedJpegFrame = frame;137214 }138215 139216 function teardown() {140217 resetAvSync();141- discardPendingJpegFrame();218+ queuedJpegFrame = null;219+ const active = activeJpegDecode;220+ activeJpegDecode = null;221+ releaseActiveJpegDecode(active, { abort: true });142222 framePresenter.clear();143223 decodeTimings.clear();144224 if (videoDecoder) {@@ -408,54 +488,16 @@ export function createVideoPipelineSession({
408488 epoch: activeVideoFormatEpoch,409489 });410490 }411- // Receipt order is authoritative. Invalidate the older async decode412- // before starting this one so a late load can never repaint the past.413- discardPendingJpegFrame({ overwritten: true });414- const sequence = ++jpegFrameSequence;491+ // Keep exactly one browser decode active and one coalesced latest payload.492+ // Receipt order is authoritative: once a newer payload is queued, the493+ // active decode may finish to release its resources but can never draw.415494 const bytes = decodeBase64(data);416495 const blob = createBlob([bytes], { type: 'image/jpeg' });417- const url = createObjectUrl(blob);418- let image;419- try {420- image = createImage();421- } catch (error) {422- revokeObjectUrl(url);423- throw error;424- }425- const jpegFrame = {426- sequence,427- image,428- url,496+ enqueueJpegFrame({497+ sequence: ++jpegFrameSequence,498+ blob,429499 receivedAt,430- released: false,431- };432- pendingJpegFrame = jpegFrame;433- image.onload = () => {434- if (!isCurrentJpegFrame(jpegFrame)) return;435- pendingJpegFrame = null;436- const drawStartedAt = now();437- try {438- context.drawImage(image, 0, 0, canvas.width, canvas.height);439- const presentedAt = now();440- presentedFrames++;441- presentationRate.record(presentedAt);442- presentationCadence.record(presentedAt);443- drawLatency.record(presentedAt - drawStartedAt, presentedAt);444- clientPresentationLatency.record(445- presentedAt - jpegFrame.receivedAt,446- presentedAt,447- );448- } finally {449- releaseJpegFrame(jpegFrame);450- }451- };452- image.onerror = () => {453- if (!isCurrentJpegFrame(jpegFrame)) return;454- pendingJpegFrame = null;455- droppedFrames++;456- releaseJpegFrame(jpegFrame);457- };458- image.src = url;500+ });459501 }460502 }461503 @@ -487,7 +529,7 @@ export function createVideoPipelineSession({
487529 decoderQueueCapacity: MAX_DECODE_QUEUE_SIZE,488530 presenterQueueDepth: hasWebCodecs489531 ? framePresenter.depth490- : Number(pendingJpegFrame !== null),532+ : Number(activeJpegDecode !== null) + Number(queuedJpegFrame !== null),491533 presenterQueueCapacity: hasWebCodecs492534 ? PRESENTER_QUEUE_CAPACITY493535 : JPEG_PRESENTER_QUEUE_CAPACITY,@@ -28,6 +28,8 @@ function harness({ hasWebCodecs = true } = {}) {
2828 const draws = [];2929 const revokedUrls = [];3030 const images = [];31+ const imageStarts = [];32+ const abortedImages = [];3133 let nextObjectUrl = 0;3234 const canvas = { width: 0, height: 0 };3335 const context = {@@ -69,10 +71,23 @@ function harness({ hasWebCodecs = true } = {}) {
6971 createObjectUrl: () => `blob:test-frame-${++nextObjectUrl}`,7072 revokeObjectUrl: (url) => revokedUrls.push(url),7173 createImage: () => {72- const image = { onload: null, onerror: null, src: null };74+ let src = null;75+ const image = {76+ onload: null,77+ onerror: null,78+ get src() { return src; },79+ set src(value) {80+ src = value;81+ imageStarts.push({ image: this, value });82+ },83+ };7384 images.push(image);7485 return image;7586 },87+ abortImage: (image) => {88+ abortedImages.push(image);89+ image.src = null;90+ },7691 requestFrame: () => 1,7792 cancelFrame: () => {},7893 setTimer: () => 2,@@ -90,6 +105,8 @@ function harness({ hasWebCodecs = true } = {}) {
90105 draws,91106 revokedUrls,92107 images,108+ imageStarts,109+ abortedImages,93110 advance(milliseconds = 1) { clock += milliseconds; },94111 };95112 }@@ -225,7 +242,7 @@ test('JPEG fallback keeps its asynchronous base64 draw behavior', () => {
225242 assert.deepEqual(subject.draws, []);226243 assert.deepEqual(subject.revokedUrls, []);227244 assert.equal(subject.pipeline.snapshot().presenterQueueDepth, 1);228- assert.equal(subject.pipeline.snapshot().presenterQueueCapacity, 1);245+ assert.equal(subject.pipeline.snapshot().presenterQueueCapacity, 2);229246 230247 subject.advance();231248 subject.images[0].onload();@@ -233,10 +250,10 @@ test('JPEG fallback keeps its asynchronous base64 draw behavior', () => {
233250 assert.deepEqual(subject.revokedUrls, ['blob:test-frame-1']);234251 assert.equal(subject.pipeline.snapshot().presentedFrames, 1);235252 assert.equal(subject.pipeline.snapshot().presenterQueueDepth, 0);236- assert.equal(subject.pipeline.snapshot().presenterQueueCapacity, 1);253+ assert.equal(subject.pipeline.snapshot().presenterQueueCapacity, 2);237254 });238255 239-test('JPEG fallback presents only the newest frame when loads complete out of order', () => {256+test('JPEG fallback suppresses an active stale decode before starting the newest frame', () => {240257 const subject = harness({ hasWebCodecs: false });241258 const payload = {242259 width: 640,@@ -250,33 +267,34 @@ test('JPEG fallback presents only the newest frame when loads complete out of or
250267 251268 subject.advance(10);252269 subject.pipeline.processFramePayload(payload);253- const newestLoad = subject.images[1].onload;254- assert.deepEqual(subject.revokedUrls, ['blob:test-frame-1']);270+ assert.equal(subject.images.length, 1);271+ assert.deepEqual(subject.revokedUrls, []);255272 const afterOverwrite = subject.pipeline.snapshot();256273 assert.equal(afterOverwrite.presentationDroppedFrames, 1);257- assert.equal(afterOverwrite.presenterQueueDepth, 1);258- assert.equal(afterOverwrite.presenterQueueCapacity, 1);274+ assert.equal(afterOverwrite.presenterQueueDepth, 2);275+ assert.equal(afterOverwrite.presenterQueueCapacity, 2);276+277+ subject.advance(5);278+ staleLoad();279+ assert.deepEqual(subject.draws, []);280+ assert.deepEqual(subject.revokedUrls, ['blob:test-frame-1']);281+ assert.equal(subject.images.length, 2);282+ assert.equal(subject.images[1].src, 'blob:test-frame-2');283+ const newestLoad = subject.images[1].onload;284+ assert.equal(subject.pipeline.snapshot().presenterQueueDepth, 1);259285 260286 subject.advance(5);261287 newestLoad();288+ staleError(new Error('stale JPEG failure'));262289 const afterNewest = subject.pipeline.snapshot();263290 assert.deepEqual(subject.draws.map(([image]) => image), [subject.images[1]]);264291 assert.equal(afterNewest.presentedFrames, 1);265292 assert.equal(afterNewest.presentPercentiles.count, 1);266- assert.equal(afterNewest.presentPercentiles.p50, 5);293+ assert.equal(afterNewest.presentPercentiles.p50, 10);267294 assert.equal(afterNewest.presentationDroppedFrames, 1);295+ assert.equal(afterNewest.droppedFrames, 0);268296 assert.equal(afterNewest.presenterQueueDepth, 0);269- assert.equal(afterNewest.presenterQueueCapacity, 1);270-271- subject.advance(100);272- staleLoad();273- staleError(new Error('stale JPEG failure'));274- const afterStaleCallbacks = subject.pipeline.snapshot();275- assert.deepEqual(subject.draws.map(([image]) => image), [subject.images[1]]);276- assert.equal(afterStaleCallbacks.presentedFrames, 1);277- assert.equal(afterStaleCallbacks.presentPercentiles.count, 1);278- assert.equal(afterStaleCallbacks.presentPercentiles.p50, 5);279- assert.equal(afterStaleCallbacks.droppedFrames, 0);297+ assert.equal(afterNewest.presenterQueueCapacity, 2);280298 assert.deepEqual(subject.revokedUrls, [281299 'blob:test-frame-1',282300 'blob:test-frame-2',@@ -295,10 +313,11 @@ test('JPEG fallback error callbacks affect only the current pending frame', () =
295313 const staleError = subject.images[0].onerror;296314 297315 subject.pipeline.processFramePayload(payload);298- const failedLoad = subject.images[1].onload;299- const currentError = subject.images[1].onerror;300316 staleError(new Error('stale JPEG failure'));301317 assert.equal(subject.pipeline.snapshot().droppedFrames, 0);318+ assert.equal(subject.images.length, 2);319+ const failedLoad = subject.images[1].onload;320+ const currentError = subject.images[1].onerror;302321 303322 currentError(new Error('current JPEG failure'));304323 failedLoad();@@ -307,7 +326,7 @@ test('JPEG fallback error callbacks affect only the current pending frame', () =
307326 assert.equal(afterFailure.presentedFrames, 0);308327 assert.equal(afterFailure.presentationDroppedFrames, 1);309328 assert.equal(afterFailure.presenterQueueDepth, 0);310- assert.equal(afterFailure.presenterQueueCapacity, 1);329+ assert.equal(afterFailure.presenterQueueCapacity, 2);311330 assert.deepEqual(subject.draws, []);312331 assert.deepEqual(subject.revokedUrls, [313332 'blob:test-frame-1',@@ -337,9 +356,64 @@ test('JPEG fallback callbacks cannot survive a session reset', () => {
337356 assert.equal(stats.droppedFrames, 0);338357 assert.equal(stats.presentPercentiles.count, 0);339358 assert.equal(stats.presenterQueueDepth, 0);340- assert.equal(stats.presenterQueueCapacity, 1);359+ assert.equal(stats.presenterQueueCapacity, 2);341360 assert.deepEqual(subject.draws, []);342361 assert.deepEqual(subject.revokedUrls, ['blob:test-frame-1']);362+ assert.deepEqual(subject.abortedImages, [subject.images[0]]);363+});364+365+test('JPEG fallback bounds delayed decoding to one active and one coalesced latest payload', () => {366+ const subject = harness({ hasWebCodecs: false });367+ const payload = {368+ width: 640,369+ height: 400,370+ data: 'AQID',371+ keyframe: true,372+ };373+ subject.pipeline.processFramePayload(payload);374+ const oldestLoad = subject.images[0].onload;375+376+ for (let frame = 2; frame <= 10; frame++) {377+ subject.advance();378+ subject.pipeline.processFramePayload(payload);379+ }380+381+ assert.equal(subject.images.length, 1);382+ assert.deepEqual(subject.imageStarts, [{383+ image: subject.images[0],384+ value: 'blob:test-frame-1',385+ }]);386+ assert.deepEqual(subject.revokedUrls, []);387+ let stats = subject.pipeline.snapshot();388+ assert.equal(stats.presentationDroppedFrames, 9);389+ assert.equal(stats.presenterQueueDepth, 2);390+ assert.equal(stats.presenterQueueCapacity, 2);391+392+ subject.advance();393+ oldestLoad();394+ assert.equal(subject.images.length, 2);395+ assert.deepEqual(subject.imageStarts, [396+ { image: subject.images[0], value: 'blob:test-frame-1' },397+ { image: subject.images[1], value: 'blob:test-frame-2' },398+ ]);399+ assert.deepEqual(subject.revokedUrls, ['blob:test-frame-1']);400+ stats = subject.pipeline.snapshot();401+ assert.equal(stats.presenterQueueDepth, 1);402+ assert.equal(stats.presenterQueueCapacity, 2);403+404+ subject.advance();405+ subject.images[1].onload();406+ stats = subject.pipeline.snapshot();407+ assert.deepEqual(subject.draws.map(([image]) => image), [subject.images[1]]);408+ assert.equal(stats.presentedFrames, 1);409+ assert.equal(stats.presentPercentiles.p50, 2);410+ assert.equal(stats.presentationDroppedFrames, 9);411+ assert.equal(stats.presenterQueueDepth, 0);412+ assert.equal(stats.presenterQueueCapacity, 2);413+ assert.deepEqual(subject.revokedUrls, [414+ 'blob:test-frame-1',415+ 'blob:test-frame-2',416+ ]);343417 });344418 345419 test('reset closes decoding state but preserves the last dimensions and codec quirk', () => {