- commit
- a8f423fcca0641592e53d39ea5f6789bcf213ce0
- parent
- 40d1df9d562378b386ea8049f43a494d893cb904
- author
- Dami <dami.etoile@gmail.com>
| A | portal/ | 498 | ++++++++++++++++++++++++++++++ |
| A | portal/ | 377 | +++++++++++++++++++++++ |
| M | portal/ | 481 | ++--------------------------- |
@@ -0,0 +1,498 @@
1+import {2+ audioMinusVideoSkewMs,3+ audioOutputTimelineFromStats,4+ exactMediaTimestampMicros,5+ isCurrentAvSyncEpoch,6+ nextAvSyncEpoch,7+ projectAudioMediaPtsMicros,8+} from './av-sync.mjs';9+import {10+ isCurrentAudioDelivery,11+ isCurrentAudioGeneration,12+ parseAudioEnvelope,13+ stageAudioTerminalState,14+ takeAudioTerminalState,15+} from './audio-envelope.mjs';16+import { newAudioSession } from './audio-ui.mjs';17+18+export const AUDIO_SAMPLE_RATE = 48000;19+20+const AUDIO_DECODER_CONFIG = Object.freeze({21+ codec: 'opus',22+ sampleRate: AUDIO_SAMPLE_RATE,23+ numberOfChannels: 2,24+});25+const MAX_AUDIO_DECODE_QUEUE_SIZE = 3;26+27+export function createAudioPipelineSession({28+ invokeCommand,29+ onUpdate = () => {},30+ resetAvSync = () => {},31+ isDisconnecting = () => false,32+ hasAudioDecoder = () => typeof globalThis.AudioDecoder === 'function',33+ hasAudioWorklet = () => typeof globalThis.AudioWorkletNode === 'function',34+ getAudioContextConstructor = () => globalThis.AudioContext || globalThis.webkitAudioContext,35+ probeDecoderSupport = (config) => globalThis.AudioDecoder.isConfigSupported(config),36+ createDecoder = (callbacks) => new globalThis.AudioDecoder(callbacks),37+ createWorkletNode = (context, name, options) => new globalThis.AudioWorkletNode(38+ context,39+ name,40+ options,41+ ),42+ createEncodedChunk = (init) => new globalThis.EncodedAudioChunk(init),43+} = {}) {44+ if (typeof invokeCommand !== 'function') throw new TypeError('invokeCommand must be a function');45+46+ let activeSession = newAudioSession();47+48+ function update() {49+ onUpdate(activeSession);50+ }51+52+ function setState(state, detail = state) {53+ activeSession.state = state;54+ activeSession.stateDetail = detail;55+ update();56+ }57+58+ function resetSyncTelemetry(session = activeSession, { recoverRing = false } = {}) {59+ if (!session) return;60+ session.audioTimeline = null;61+ session.avSyncEpoch = nextAvSyncEpoch(session.avSyncEpoch);62+ try {63+ session.workletNode?.port.postMessage({64+ type: recoverRing ? 'recover' : 'av-sync-epoch',65+ avSyncEpoch: session.avSyncEpoch,66+ });67+ } catch (_) {}68+ }69+70+ function sampleSkew(videoPtsMicros, presentationTimeMs) {71+ const session = activeSession;72+ if (73+ exactMediaTimestampMicros(videoPtsMicros) === null74+ || !session75+ || session.failed76+ || !session.audioTimeline77+ || !session.available78+ || session.context?.state !== 'running'79+ || typeof session.context.getOutputTimestamp !== 'function'80+ ) return null;81+82+ let outputTimestamp;83+ try {84+ outputTimestamp = session.context.getOutputTimestamp();85+ } catch (_) {86+ return null;87+ }88+ const audioPtsMicros = projectAudioMediaPtsMicros(89+ session.audioTimeline,90+ outputTimestamp,91+ presentationTimeMs,92+ );93+ return audioMinusVideoSkewMs(audioPtsMicros, videoPtsMicros);94+ }95+96+ function resetTelemetry(session = activeSession) {97+ session.packetsReceived = 0;98+ session.decoderDropped = 0;99+ session.bufferedFrames = 0;100+ session.underflows = 0;101+ session.underflowDurationMicros = 0;102+ session.silentDurationMicros = 0;103+ session.transportDropped = 0;104+ session.frontendDropped = 0;105+ resetAvSync(session);106+ update();107+ }108+109+ function acknowledge(generation, deliveryId) {110+ void invokeCommand('iroh_client_ack_audio', { generation, deliveryId }).catch((error) => {111+ console.warn('audio acknowledgment failed:', error);112+ });113+ }114+115+ function requestNativeStop(session) {116+ const generation = session?.expectedGeneration;117+ if (session !== activeSession118+ || !Number.isSafeInteger(generation)119+ || generation < 0120+ || session.stopRequested) return;121+ session.stopRequested = true;122+ void invokeCommand('iroh_client_stop_audio', { expectedGeneration: generation }).catch((error) => {123+ console.warn('audio stop failed:', error);124+ });125+ }126+127+ function disable(detail, session = activeSession) {128+ if (!session || session !== activeSession) return;129+ resetAvSync(session);130+ session.failed = true;131+ session.failureDetail = detail;132+ session.available = false;133+ if (session.decoder) {134+ try { session.decoder.close(); } catch (_) {}135+ session.decoder = null;136+ }137+ session.messageTracker.clear();138+ try { session.workletNode?.port.postMessage({ type: 'clear' }); } catch (_) {}139+ requestNativeStop(session);140+ setState('error', detail);141+ }142+143+ function configureDecoder(session = activeSession) {144+ if (session.decoder) {145+ try { session.decoder.close(); } catch (_) {}146+ }147+ session.decoder = createDecoder({148+ output: (audioData) => {149+ try {150+ if (session !== activeSession || session.failed) return;151+ const workletNode = session?.workletNode;152+ if (!workletNode || audioData.numberOfChannels !== 2) {153+ throw new Error(`unexpected decoded audio channel count: ${audioData.numberOfChannels}`);154+ }155+ const timestampMicros = exactMediaTimestampMicros(audioData.timestamp);156+ if (timestampMicros === null) throw new Error('decoded audio has an invalid media timestamp');157+ // Reserve ownership before allocating/copying PCM. The MessagePort has158+ // no useful queue-depth API, so this makes overload latest-frame-wins159+ // at a hard ceiling of three decoded messages.160+ const messageId = session.messageTracker.reserve();161+ if (messageId === null) {162+ update();163+ return;164+ }165+ const channels = [];166+ const transfers = [];167+ try {168+ for (let planeIndex = 0; planeIndex < 2; planeIndex++) {169+ const channel = new Float32Array(audioData.numberOfFrames);170+ audioData.copyTo(channel, { planeIndex, format: 'f32-planar' });171+ channels.push(channel);172+ transfers.push(channel.buffer);173+ }174+ workletNode.port.postMessage({175+ type: 'samples',176+ id: messageId,177+ channels,178+ timestampMicros,179+ }, transfers);180+ } catch (error) {181+ session.messageTracker.accept(messageId);182+ throw error;183+ }184+ } catch (error) {185+ disable(`Decoded audio output failed: ${error}`, session);186+ } finally {187+ audioData.close();188+ }189+ },190+ error: (error) => {191+ disable(`Opus decoder failed: ${error}`, session);192+ },193+ });194+ session.decoder.configure(AUDIO_DECODER_CONFIG);195+ }196+197+ function primeOutputForActivation() {198+ const session = activeSession;199+ const AudioContextConstructor = getAudioContextConstructor();200+ if (typeof AudioContextConstructor !== 'function') return;201+ if (!session.context || session.context.state === 'closed') {202+ try {203+ session.context = new AudioContextConstructor({204+ latencyHint: 'interactive',205+ sampleRate: AUDIO_SAMPLE_RATE,206+ });207+ } catch (_) {208+ return;209+ }210+ }211+ // This function is deliberately synchronous and is called before the first212+ // await in a click/keyboard connect handler, preserving WebKit user activation.213+ void session.context.resume().catch(() => {});214+ }215+216+ async function initialize(session) {217+ if (session.decoder) {218+ try { session.decoder.close(); } catch (_) {}219+ session.decoder = null;220+ }221+ if (session.workletNode) {222+ try { session.workletNode.disconnect(); } catch (_) {}223+ session.workletNode = null;224+ }225+ session.messageTracker.clear();226+ if (!hasAudioDecoder()) {227+ return { supported: false, error: 'WebCodecs AudioDecoder is unavailable' };228+ }229+ if (!hasAudioWorklet()) {230+ return { supported: false, error: 'AudioWorklet is unavailable' };231+ }232+ let support;233+ try {234+ support = await probeDecoderSupport(AUDIO_DECODER_CONFIG);235+ } catch (error) {236+ return { supported: false, error: `Opus capability probe failed: ${error}` };237+ }238+ if (!support.supported) {239+ return { supported: false, error: 'WebCodecs Opus decoding is unsupported' };240+ }241+ if (!session.context) {242+ return { supported: false, error: 'Web Audio output is unavailable' };243+ }244+ try {245+ if (session.context.sampleRate !== AUDIO_SAMPLE_RATE) {246+ throw new Error(`audio output opened at ${session.context.sampleRate} Hz instead of 48000 Hz`);247+ }248+ await session.context.audioWorklet.addModule(new URL('./audio-worklet.js', import.meta.url));249+ session.workletNode = createWorkletNode(session.context, 'sigil-audio-processor', {250+ numberOfInputs: 0,251+ numberOfOutputs: 1,252+ outputChannelCount: [2],253+ });254+ session.workletNode.port.onmessage = (event) => {255+ if (event.data?.type === 'accepted') {256+ session.messageTracker.accept(event.data.id);257+ if (session === activeSession) update();258+ } else if (event.data?.type === 'stats') {259+ if (!isCurrentAvSyncEpoch(event.data.avSyncEpoch, session.avSyncEpoch)) return;260+ session.workletDroppedFrames = Number.isSafeInteger(event.data.droppedFrames)261+ && event.data.droppedFrames >= 0262+ ? event.data.droppedFrames : session.workletDroppedFrames;263+ session.workletRecoveryDiscardedFrames = Number.isSafeInteger(264+ event.data.recoveryDiscardedFrames,265+ ) && event.data.recoveryDiscardedFrames >= 0266+ ? event.data.recoveryDiscardedFrames : session.workletRecoveryDiscardedFrames;267+ session.audioTimeline = audioOutputTimelineFromStats(event.data);268+ if (session !== activeSession) return;269+ session.bufferedFrames = Number.isSafeInteger(event.data.bufferedFrames)270+ ? event.data.bufferedFrames : session.bufferedFrames;271+ session.underflows = Number.isSafeInteger(event.data.underflows)272+ ? event.data.underflows : session.underflows;273+ session.underflowDurationMicros = Number.isFinite(event.data.underflowDurationMicros)274+ && event.data.underflowDurationMicros >= 0275+ ? event.data.underflowDurationMicros : session.underflowDurationMicros;276+ session.silentDurationMicros = Number.isFinite(event.data.silentDurationMicros)277+ && event.data.silentDurationMicros >= 0278+ ? event.data.silentDurationMicros : session.silentDurationMicros;279+ if (session.available && !session.muted) {280+ setState(event.data.started ? 'playing' : 'priming', 'bounded Opus playback');281+ } else {282+ update();283+ }284+ } else if (event.data?.type === 'error') {285+ disable(`AudioWorklet failed: ${event.data.error}`, session);286+ }287+ };288+ session.workletNode.port.onmessageerror = () => {289+ disable('AudioWorklet returned an unreadable message', session);290+ };291+ session.workletNode.onprocessorerror = () => {292+ disable('AudioWorklet processor stopped unexpectedly', session);293+ };294+ session.workletNode.connect(session.context.destination);295+ session.workletNode.port.postMessage({ type: 'mute', muted: session.muted });296+ configureDecoder(session);297+ await session.context.resume();298+ setState(299+ session.context.state === 'running' ? 'negotiating' : 'blocked',300+ session.context.state === 'running'301+ ? 'Opus output ready'302+ : 'WebKit suspended audio output; activate the audio button to retry',303+ );304+ return { supported: true, error: null };305+ } catch (error) {306+ await teardown(false);307+ return { supported: false, error: `Audio initialization failed: ${error}` };308+ }309+ }310+311+ async function teardown(resetStatus = true) {312+ const session = activeSession;313+ resetAvSync(session);314+ const decoder = session.decoder;315+ const workletNode = session.workletNode;316+ const context = session.context;317+ // Replace the active owner before closing resources so delayed callbacks318+ // cannot mutate the disconnected UI or a successor session. Seed the next319+ // dormant session with the user's mute preference across reconnects.320+ activeSession = newAudioSession({ muted: session.muted });321+ session.expectedGeneration = null;322+ session.channel = null;323+ session.available = false;324+ session.messageTracker.clear();325+ session.decoder = null;326+ session.workletNode = null;327+ session.context = null;328+ if (decoder) {329+ try { decoder.close(); } catch (_) {}330+ }331+ if (workletNode) {332+ try { workletNode.port.postMessage({ type: 'clear' }); } catch (_) {}333+ try { workletNode.disconnect(); } catch (_) {}334+ }335+ if (context && context.state !== 'closed') {336+ try { await context.close(); } catch (_) {}337+ }338+ if (resetStatus) update();339+ }340+341+ async function toggleMute() {342+ const session = activeSession;343+ if (!session.available) return;344+ session.muted = !session.muted;345+ try {346+ if (!session.muted && session.context?.state !== 'running') await session.context?.resume();347+ session.workletNode?.port.postMessage({ type: 'mute', muted: session.muted });348+ if (!session.muted && session.context?.state !== 'running') {349+ setState('blocked', 'WebKit suspended audio output; click the audio button to retry');350+ } else {351+ update();352+ }353+ } catch (error) {354+ setState('error', `Audio output activation failed: ${error}`);355+ }356+ }357+358+ function prepareForConnection() {359+ activeSession = newAudioSession({ muted: activeSession.muted });360+ activeSession.acceptsNativeEvents = true;361+ // Keep this synchronous and before createConnectionAttempt's first await so362+ // WebKit observes the originating click/keyboard user activation.363+ primeOutputForActivation();364+ }365+366+ function beginAttempt() {367+ const session = activeSession;368+ resetTelemetry(session);369+ return session;370+ }371+372+ async function createAttemptChannel(Channel, session = activeSession) {373+ const support = await initialize(session);374+ if (!support.supported) setState('unavailable', support.error);375+ session.channel = new Channel((message) => handleBinaryMessage(message, session));376+ return { session, support, channel: session.channel };377+ }378+379+ function releaseAttempt(session) {380+ if (session && session !== activeSession) session.channel = null;381+ }382+383+ async function acceptConnectedResult(result, { session, support }) {384+ session.expectedGeneration = Number.isSafeInteger(result.audio_generation)385+ && result.audio_generation > 0386+ ? result.audio_generation : null;387+ const pendingTerminal = takeAudioTerminalState(388+ session.pendingTerminalStates,389+ session.expectedGeneration,390+ );391+ if (pendingTerminal !== null) {392+ session.failed = true;393+ session.failureDetail = pendingTerminal.error || 'Audio connection ended';394+ }395+ session.available = result.audio_available === true;396+ if (session.available && session.failed) {397+ disable(session.failureDetail || 'Audio output failed', session);398+ } else if (session.available) {399+ setState(400+ session.context?.state === 'running' ? 'priming' : 'blocked',401+ session.context?.state === 'running'402+ ? 'Waiting for bounded Opus prebuffer'403+ : 'WebKit suspended audio output; activate the audio button to retry',404+ );405+ } else {406+ await teardown(false);407+ setState('unavailable', result.audio_error || support.error || 'host audio unavailable');408+ }409+ }410+411+ function handleBinaryMessage(message, session) {412+ let packet;413+ try {414+ packet = parseAudioEnvelope(message);415+ } catch (error) {416+ console.error('invalid audio packet:', error);417+ disable(`Audio packet failed: ${error}`, session);418+ return;419+ }420+421+ // Every valid delivery is released using the token embedded in that exact422+ // message. A delayed delivery from an old connection is acknowledged but423+ // cannot enter the current decoder.424+ acknowledge(packet.generation, packet.deliveryId);425+ if (session !== activeSession426+ || session.failed427+ || !isCurrentAudioDelivery(packet, session.expectedGeneration)) return;428+429+ try {430+ session.packetsReceived++;431+ if (!session.decoder || session.decoder.state === 'closed') {432+ session.decoderDropped++;433+ update();434+ return;435+ }436+ const queuedAudioPackets = session.decoder.decodeQueueSize;437+ if (packet.discontinuity || queuedAudioPackets >= MAX_AUDIO_DECODE_QUEUE_SIZE) {438+ resetAvSync(session, { recoverRing: true });439+ session.decoderDropped += queuedAudioPackets;440+ configureDecoder(session);441+ }442+ session.decoder.decode(createEncodedChunk({443+ type: 'key',444+ timestamp: packet.ptsMicros,445+ duration: (packet.frameSamples * 1_000_000) / packet.sampleRate,446+ data: packet.data,447+ }));448+ update();449+ } catch (error) {450+ console.error('undecodable audio packet:', error);451+ disable(`Audio packet failed: ${error}`, session);452+ }453+ }454+455+ function handleNativeState(payload) {456+ const session = activeSession;457+ if (isDisconnecting()458+ || !session.acceptsNativeEvents459+ || payload?.available !== false) return;460+ if (session.expectedGeneration === null) {461+ try {462+ stageAudioTerminalState(session.pendingTerminalStates, payload);463+ } catch (error) {464+ console.error('invalid pre-connect audio terminal state:', error);465+ }466+ return;467+ }468+ if (!isCurrentAudioGeneration(payload?.generation, session.expectedGeneration)) return;469+ disable(payload.error || 'Audio connection ended', session);470+ }471+472+ function handleNativeStats(payload) {473+ const session = activeSession;474+ if (!isCurrentAudioGeneration(payload?.generation, session.expectedGeneration)) return;475+ session.transportDropped = Number.isSafeInteger(payload?.sequence_dropped_total)476+ ? payload.sequence_dropped_total : session.transportDropped;477+ session.frontendDropped = Number.isSafeInteger(payload?.frontend_dropped_total)478+ ? payload.frontend_dropped_total : session.frontendDropped;479+ update();480+ }481+482+ return Object.freeze({483+ acceptConnectedResult,484+ beginAttempt,485+ createAttemptChannel,486+ handleNativeState,487+ handleNativeStats,488+ prepareForConnection,489+ releaseAttempt,490+ resetSyncTelemetry,491+ sampleSkew,492+ teardown,493+ toggleMute,494+ get session() {495+ return activeSession;496+ },497+ });498+}@@ -0,0 +1,377 @@
1+import assert from 'node:assert/strict';2+import test from 'node:test';3+4+import {5+ AUDIO_SAMPLE_RATE,6+ createAudioPipelineSession,7+} from './audio-pipeline.mjs';8+import {9+ AUDIO_CHANNEL_PREFIX_LENGTH,10+ AUDIO_ENVELOPE_HEADER_LENGTH,11+} from './audio-envelope.mjs';12+13+const PACKET_BASE = AUDIO_CHANNEL_PREFIX_LENGTH;14+15+function envelope({16+ generation = 7n,17+ deliveryId = 11n,18+ flags = 0,19+ pts = 120000n,20+} = {}) {21+ const payload = Uint8Array.of(0xf8, 0xff, 0xfe);22+ const buffer = new ArrayBuffer(23+ AUDIO_CHANNEL_PREFIX_LENGTH + AUDIO_ENVELOPE_HEADER_LENGTH + payload.length,24+ );25+ const bytes = new Uint8Array(buffer);26+ const view = new DataView(buffer);27+ bytes.set([0x53, 0x47, 0x41, 0x43]);28+ view.setUint16(4, 1, false);29+ view.setUint16(6, AUDIO_CHANNEL_PREFIX_LENGTH, false);30+ view.setBigUint64(8, generation, false);31+ view.setBigUint64(16, deliveryId, false);32+ bytes.set([0x53, 0x47, 0x41, 0x31], PACKET_BASE);33+ view.setUint16(PACKET_BASE + 4, 1, false);34+ bytes[PACKET_BASE + 6] = AUDIO_ENVELOPE_HEADER_LENGTH;35+ bytes[PACKET_BASE + 7] = 1;36+ bytes[PACKET_BASE + 8] = flags;37+ bytes[PACKET_BASE + 9] = 2;38+ view.setUint32(PACKET_BASE + 12, AUDIO_SAMPLE_RATE, false);39+ view.setUint16(PACKET_BASE + 16, 960, false);40+ view.setUint32(PACKET_BASE + 20, payload.length, false);41+ view.setBigUint64(PACKET_BASE + 24, 42n, false);42+ view.setBigUint64(PACKET_BASE + 32, 123456n, false);43+ view.setBigInt64(PACKET_BASE + 40, pts, false);44+ bytes.set(payload, PACKET_BASE + AUDIO_ENVELOPE_HEADER_LENGTH);45+ return buffer;46+}47+48+function harness({49+ disconnecting = () => false,50+ hasAudioDecoder = () => true,51+ hasAudioWorklet = () => true,52+} = {}) {53+ const invokes = [];54+ const updates = [];55+ const avResets = [];56+ const decoders = [];57+ const worklets = [];58+ const contexts = [];59+ const chunks = [];60+61+ class FakeContext {62+ constructor(options) {63+ this.options = options;64+ this.sampleRate = options.sampleRate;65+ this.state = 'suspended';66+ this.destination = {};67+ this.modules = [];68+ this.resumeCalls = 0;69+ this.closeCalls = 0;70+ this.audioWorklet = {71+ addModule: async (url) => { this.modules.push(url); },72+ };73+ contexts.push(this);74+ }75+76+ async resume() {77+ this.resumeCalls++;78+ this.state = 'running';79+ }80+81+ async close() {82+ this.closeCalls++;83+ this.state = 'closed';84+ }85+86+ getOutputTimestamp() {87+ return { contextTime: 4, performanceTime: 1000 };88+ }89+ }90+91+ class FakeChannel {92+ constructor(callback) {93+ this.callback = callback;94+ }95+ }96+97+ const pipeline = createAudioPipelineSession({98+ invokeCommand: async (command, args) => { invokes.push({ command, args }); },99+ onUpdate: (session) => updates.push(session),100+ resetAvSync: (...args) => {101+ avResets.push(args);102+ pipeline.resetSyncTelemetry(...args);103+ },104+ isDisconnecting: disconnecting,105+ hasAudioDecoder,106+ hasAudioWorklet,107+ getAudioContextConstructor: () => FakeContext,108+ probeDecoderSupport: async (config) => ({ supported: config.codec === 'opus' }),109+ createDecoder: (callbacks) => {110+ const decoder = {111+ callbacks,112+ state: 'configured',113+ decodeQueueSize: 0,114+ configured: null,115+ decoded: [],116+ closeCalls: 0,117+ configure(config) { this.configured = config; },118+ decode(chunk) { this.decoded.push(chunk); },119+ close() { this.closeCalls++; this.state = 'closed'; },120+ };121+ decoders.push(decoder);122+ return decoder;123+ },124+ createWorkletNode: (context, name, options) => {125+ const node = {126+ context,127+ name,128+ options,129+ connected: null,130+ disconnectCalls: 0,131+ port: {132+ messages: [],133+ postMessage(message, transfers) { this.messages.push({ message, transfers }); },134+ onmessage: null,135+ onmessageerror: null,136+ },137+ connect(destination) { this.connected = destination; },138+ disconnect() { this.disconnectCalls++; },139+ onprocessorerror: null,140+ };141+ worklets.push(node);142+ return node;143+ },144+ createEncodedChunk: (init) => {145+ const chunk = { ...init };146+ chunks.push(chunk);147+ return chunk;148+ },149+ });150+151+ return {152+ pipeline,153+ FakeChannel,154+ invokes,155+ updates,156+ avResets,157+ decoders,158+ worklets,159+ contexts,160+ chunks,161+ };162+}163+164+async function initializeAttempt(fixture) {165+ fixture.pipeline.prepareForConnection();166+ const primedSession = fixture.pipeline.session;167+ fixture.pipeline.beginAttempt();168+ const attempt = await fixture.pipeline.createAttemptChannel(fixture.FakeChannel, primedSession);169+ return { ...attempt, primedSession };170+}171+172+test('connection preparation synchronously primes WebKit audio and initializes bounded Opus resources', async () => {173+ const fixture = harness();174+ const initial = fixture.pipeline.session;175+ initial.muted = true;176+177+ fixture.pipeline.prepareForConnection();178+ const session = fixture.pipeline.session;179+ assert.notEqual(session, initial);180+ assert.equal(session.muted, true);181+ assert.equal(session.acceptsNativeEvents, true);182+ assert.equal(fixture.contexts.length, 1);183+ assert.equal(fixture.contexts[0].resumeCalls, 1);184+185+ fixture.pipeline.beginAttempt();186+ const attempt = await fixture.pipeline.createAttemptChannel(fixture.FakeChannel, session);187+ assert.equal(attempt.session, session);188+ assert.deepEqual(attempt.support, { supported: true, error: null });189+ assert.equal(attempt.channel, session.channel);190+ assert.equal(fixture.decoders[0].configured.codec, 'opus');191+ assert.equal(fixture.decoders[0].configured.sampleRate, AUDIO_SAMPLE_RATE);192+ assert.equal(fixture.worklets[0].name, 'sigil-audio-processor');193+ assert.deepEqual(fixture.worklets[0].port.messages[0].message, { type: 'mute', muted: true });194+ assert.equal(session.state, 'negotiating');195+ assert.equal(session.stateDetail, 'Opus output ready');196+});197+198+test('capability checks preserve the exact WebCodecs and AudioWorklet fallback errors', async () => {199+ const missingDecoder = harness({ hasAudioDecoder: () => false });200+ missingDecoder.pipeline.prepareForConnection();201+ let session = missingDecoder.pipeline.beginAttempt();202+ let attempt = await missingDecoder.pipeline.createAttemptChannel(203+ missingDecoder.FakeChannel,204+ session,205+ );206+ assert.deepEqual(attempt.support, {207+ supported: false,208+ error: 'WebCodecs AudioDecoder is unavailable',209+ });210+211+ const missingWorklet = harness({ hasAudioWorklet: () => false });212+ missingWorklet.pipeline.prepareForConnection();213+ session = missingWorklet.pipeline.beginAttempt();214+ attempt = await missingWorklet.pipeline.createAttemptChannel(missingWorklet.FakeChannel, session);215+ assert.deepEqual(attempt.support, {216+ supported: false,217+ error: 'AudioWorklet is unavailable',218+ });219+});220+221+test('deliveries always acknowledge but only the current generation enters the bounded decoder', async () => {222+ const fixture = harness();223+ const attempt = await initializeAttempt(fixture);224+ await fixture.pipeline.acceptConnectedResult({225+ audio_generation: 7,226+ audio_available: true,227+ audio_error: null,228+ }, attempt);229+230+ attempt.channel.callback(envelope({ generation: 8n, deliveryId: 80n }));231+ attempt.channel.callback(envelope({ generation: 7n, deliveryId: 70n }));232+ await Promise.resolve();233+ assert.deepEqual(fixture.invokes.slice(0, 2), [234+ { command: 'iroh_client_ack_audio', args: { generation: 8, deliveryId: 80 } },235+ { command: 'iroh_client_ack_audio', args: { generation: 7, deliveryId: 70 } },236+ ]);237+ assert.equal(fixture.decoders[0].decoded.length, 1);238+ assert.equal(fixture.chunks[0].timestamp, 120000);239+ assert.equal(fixture.chunks[0].duration, 20000);240+241+ let audioDataClosed = false;242+ fixture.decoders[0].callbacks.output({243+ numberOfChannels: 2,244+ numberOfFrames: 2,245+ timestamp: 120000,246+ copyTo: (channel, { planeIndex }) => channel.fill(planeIndex + 1),247+ close: () => { audioDataClosed = true; },248+ });249+ const samples = fixture.worklets[0].port.messages.at(-1);250+ assert.equal(samples.message.type, 'samples');251+ assert.deepEqual([...samples.message.channels[0]], [1, 1]);252+ assert.deepEqual([...samples.message.channels[1]], [2, 2]);253+ assert.equal(samples.transfers.length, 2);254+ assert.equal(audioDataClosed, true);255+256+ fixture.decoders[0].decodeQueueSize = 3;257+ attempt.channel.callback(envelope({ generation: 7n, deliveryId: 71n }));258+ assert.equal(fixture.decoders.length, 2);259+ assert.equal(fixture.decoders[0].closeCalls, 1);260+ assert.equal(attempt.session.decoderDropped, 3);261+ assert.equal(fixture.decoders[1].decoded.length, 1);262+ assert.equal(fixture.avResets.at(-1)[1].recoverRing, true);263+ assert.deepEqual(fixture.worklets[0].port.messages.at(-1).message, {264+ type: 'recover',265+ avSyncEpoch: attempt.session.avSyncEpoch,266+ });267+});268+269+test('worklet telemetry is epoch-gated and projects signed audio/video skew', async () => {270+ const fixture = harness();271+ const attempt = await initializeAttempt(fixture);272+ await fixture.pipeline.acceptConnectedResult({273+ audio_generation: 7,274+ audio_available: true,275+ audio_error: null,276+ }, attempt);277+ const session = attempt.session;278+ const worklet = fixture.worklets[0];279+280+ worklet.port.onmessage({ data: {281+ type: 'stats',282+ avSyncEpoch: session.avSyncEpoch + 1,283+ bufferedFrames: 99,284+ } });285+ assert.equal(session.bufferedFrames, 0);286+287+ worklet.port.onmessage({ data: {288+ type: 'stats',289+ avSyncEpoch: session.avSyncEpoch,290+ droppedFrames: 11,291+ recoveryDiscardedFrames: 12,292+ renderedMediaEndPtsMicros: 2_000_000,293+ outputContextTimeEndSeconds: 4,294+ bufferedFrames: 960,295+ underflows: 2,296+ underflowDurationMicros: 10_000,297+ silentDurationMicros: 20_000,298+ started: true,299+ } });300+ assert.equal(session.state, 'playing');301+ assert.equal(session.workletDroppedFrames, 11);302+ assert.equal(session.workletRecoveryDiscardedFrames, 12);303+ assert.equal(session.bufferedFrames, 960);304+ assert.equal(fixture.pipeline.sampleSkew(1_975_000, 1000), 25);305+306+ fixture.pipeline.resetSyncTelemetry(session, { recoverRing: true });307+ assert.equal(session.audioTimeline, null);308+ assert.deepEqual(worklet.port.messages.at(-1).message, {309+ type: 'recover',310+ avSyncEpoch: session.avSyncEpoch,311+ });312+ assert.equal(fixture.pipeline.sampleSkew(1_975_000, 1000), null);313+});314+315+test('native terminal state is staged before connect and teardown isolates delayed callbacks', async () => {316+ const fixture = harness();317+ fixture.pipeline.prepareForConnection();318+ fixture.pipeline.handleNativeState({ generation: 9, available: false, error: 'ended early' });319+ assert.equal(fixture.pipeline.session.pendingTerminalStates.length, 1);320+ const session = fixture.pipeline.beginAttempt();321+ const attempt = await fixture.pipeline.createAttemptChannel(fixture.FakeChannel, session);322+ const oldSession = attempt.session;323+ const oldWorklet = fixture.worklets[0];324+ await fixture.pipeline.acceptConnectedResult({325+ audio_generation: 9,326+ audio_available: true,327+ audio_error: null,328+ }, attempt);329+ await Promise.resolve();330+ assert.equal(oldSession.failed, true);331+ assert.equal(oldSession.failureDetail, 'ended early');332+ assert.deepEqual(fixture.invokes.at(-1), {333+ command: 'iroh_client_stop_audio',334+ args: { expectedGeneration: 9 },335+ });336+337+ oldSession.muted = true;338+ await fixture.pipeline.teardown();339+ const successor = fixture.pipeline.session;340+ assert.notEqual(successor, oldSession);341+ assert.equal(successor.muted, true);342+ assert.equal(fixture.contexts[0].closeCalls, 1);343+ assert.equal(oldWorklet.disconnectCalls, 1);344+ const updateCount = fixture.updates.length;345+ oldWorklet.port.onmessage({ data: { type: 'accepted', id: 1 } });346+ assert.equal(fixture.updates.length, updateCount);347+});348+349+test('native stats and mute toggles update only the exact active generation', async () => {350+ const fixture = harness();351+ const attempt = await initializeAttempt(fixture);352+ await fixture.pipeline.acceptConnectedResult({353+ audio_generation: 7,354+ audio_available: true,355+ audio_error: null,356+ }, attempt);357+ fixture.pipeline.handleNativeStats({358+ generation: 6,359+ sequence_dropped_total: 99,360+ frontend_dropped_total: 98,361+ });362+ assert.equal(attempt.session.transportDropped, 0);363+ fixture.pipeline.handleNativeStats({364+ generation: 7,365+ sequence_dropped_total: 4,366+ frontend_dropped_total: 5,367+ });368+ assert.equal(attempt.session.transportDropped, 4);369+ assert.equal(attempt.session.frontendDropped, 5);370+371+ await fixture.pipeline.toggleMute();372+ assert.equal(attempt.session.muted, true);373+ assert.deepEqual(fixture.worklets[0].port.messages.at(-1).message, {374+ type: 'mute',375+ muted: true,376+ });377+});@@ -29,23 +29,12 @@ import {
2929 } from './frame-session.mjs';3030 import { newPointerSession, parsePointerFeedbackMessage } from './pointer-feedback.mjs';3131 import { newConnectionState } from './connection-state.mjs';32+import { formatSignedMilliseconds } from './av-sync.mjs';33+import { audioButtonPresentation } from './audio-ui.mjs';3234 import {33- audioMinusVideoSkewMs,34- audioOutputTimelineFromStats,35- exactMediaTimestampMicros,36- formatSignedMilliseconds,37- isCurrentAvSyncEpoch,38- nextAvSyncEpoch,39- projectAudioMediaPtsMicros,40-} from './av-sync.mjs';41-import {42- isCurrentAudioDelivery,43- isCurrentAudioGeneration,44- parseAudioEnvelope,45- stageAudioTerminalState,46- takeAudioTerminalState,47-} from './audio-envelope.mjs';48-import { audioButtonPresentation, newAudioSession } from './audio-ui.mjs';35+ AUDIO_SAMPLE_RATE,36+ createAudioPipelineSession,37+} from './audio-pipeline.mjs';4938 import {5039 ControllerActivationGate,5140 ControllerActionRepeater,@@ -94,7 +83,6 @@ let activeFrameChannel = null;
9483 let activeFrameGeneration = null;9584 let activeFrameSession = null;9685 let activePointerSession = null;97-let activeAudioSession = newAudioSession();9886 let controllerActivationInProgress = false;9987 let controlTransitionInProgress = false;10088 let controlTransitionGeneration = 0;@@ -102,13 +90,6 @@ let nativeCursorCommand = Promise.resolve();
10290 let browserPointerLockRequired = true;10391 const controllerActivationGate = new ControllerActivationGate();10492 105-const AUDIO_DECODER_CONFIG = Object.freeze({106- codec: 'opus',107- sampleRate: 48000,108- numberOfChannels: 2,109-});110-const MAX_AUDIO_DECODE_QUEUE_SIZE = 3;111-11293 function applyPointerPositionFeedback(message, session) {11394 if (session !== activePointerSession || !connectionState.inputCapabilities.pointerPositionFeedback) return;11495 const surface = currentPointerSurfaceSize();@@ -152,6 +133,13 @@ function handlePointerPositionFeedback(message, session) {
152133 153134 const invoke = (...args) => window.__TAURI__.core.invoke(...args);154135 const listen = (...args) => window.__TAURI__.event.listen(...args);136+let videoPipeline = null;137+const audioPipeline = createAudioPipelineSession({138+ invokeCommand: invoke,139+ onUpdate: updateAudioUI,140+ resetAvSync: (...args) => videoPipeline?.resetAvSync(...args),141+ isDisconnecting: () => connectionState.disconnecting,142+});155143 const adaptiveFeedbackPublisher = new AdaptiveFeedbackPublisher({ invokeCommand: invoke });156144 let adaptiveFeedbackAvailable = false;157145 let adaptiveFeedbackError = null;@@ -386,7 +374,7 @@ async function introConnect() {
386374 387375 // ─── Client ───────────────────────────────────────────────────────────────────388376 function updateAudioUI() {389- const session = activeAudioSession;377+ const session = audioPipeline.session;390378 const toggle = document.getElementById('audio-toggle');391379 const status = document.getElementById('stream-audio-state');392380 const webviewReceived = document.getElementById('stream-audio-webview-received-packets');@@ -425,13 +413,13 @@ function updateAudioUI() {
425413 pcmHandoffDropped.textContent = `${session.messageTracker.droppedMessages} blocks`;426414 }427415 if (ringOverflow) {428- ringOverflow.textContent = `${(session.workletDroppedFrames * 1000 / AUDIO_DECODER_CONFIG.sampleRate).toFixed(1)} ms`;416+ ringOverflow.textContent = `${(session.workletDroppedFrames * 1000 / AUDIO_SAMPLE_RATE).toFixed(1)} ms`;429417 }430418 if (ringRecoveryDiscarded) {431- ringRecoveryDiscarded.textContent = `${(session.workletRecoveryDiscardedFrames * 1000 / AUDIO_DECODER_CONFIG.sampleRate).toFixed(1)} ms`;419+ ringRecoveryDiscarded.textContent = `${(session.workletRecoveryDiscardedFrames * 1000 / AUDIO_SAMPLE_RATE).toFixed(1)} ms`;432420 }433421 if (ringBuffered) {434- ringBuffered.textContent = `${(session.bufferedFrames * 1000 / AUDIO_DECODER_CONFIG.sampleRate).toFixed(1)} ms`;422+ ringBuffered.textContent = `${(session.bufferedFrames * 1000 / AUDIO_SAMPLE_RATE).toFixed(1)} ms`;435423 }436424 if (underruns) {437425 underruns.textContent = `${session.underflows} events / ${(session.underflowDurationMicros / 1000).toFixed(1)} ms`;@@ -439,243 +427,8 @@ function updateAudioUI() {
439427 if (silence) silence.textContent = `${(session.silentDurationMicros / 1000).toFixed(1)} ms`;440428 }441429 442-function setAudioState(state, detail = state) {443- activeAudioSession.state = state;444- activeAudioSession.stateDetail = detail;445- updateAudioUI();446-}447-448-function resetAudioTelemetry(session = activeAudioSession) {449- session.packetsReceived = 0;450- session.decoderDropped = 0;451- session.bufferedFrames = 0;452- session.underflows = 0;453- session.underflowDurationMicros = 0;454- session.silentDurationMicros = 0;455- session.transportDropped = 0;456- session.frontendDropped = 0;457- videoPipeline.resetAvSync(session);458- updateAudioUI();459-}460-461-function configureAudioDecoder(session = activeAudioSession) {462- if (session.decoder) {463- try { session.decoder.close(); } catch (_) {}464- }465- session.decoder = new AudioDecoder({466- output: (audioData) => {467- try {468- if (session !== activeAudioSession || session.failed) return;469- const workletNode = session?.workletNode;470- if (!workletNode || audioData.numberOfChannels !== 2) {471- throw new Error(`unexpected decoded audio channel count: ${audioData.numberOfChannels}`);472- }473- const timestampMicros = exactMediaTimestampMicros(audioData.timestamp);474- if (timestampMicros === null) throw new Error('decoded audio has an invalid media timestamp');475- // Reserve ownership before allocating/copying PCM. The MessagePort has476- // no useful queue-depth API, so this makes overload latest-frame-wins477- // at a hard ceiling of three decoded messages.478- const messageId = session.messageTracker.reserve();479- if (messageId === null) {480- updateAudioUI();481- return;482- }483- const channels = [];484- const transfers = [];485- try {486- for (let planeIndex = 0; planeIndex < 2; planeIndex++) {487- const channel = new Float32Array(audioData.numberOfFrames);488- audioData.copyTo(channel, { planeIndex, format: 'f32-planar' });489- channels.push(channel);490- transfers.push(channel.buffer);491- }492- workletNode.port.postMessage({493- type: 'samples',494- id: messageId,495- channels,496- timestampMicros,497- }, transfers);498- } catch (error) {499- session.messageTracker.accept(messageId);500- throw error;501- }502- } catch (error) {503- disableAudioForSession(`Decoded audio output failed: ${error}`, session);504- } finally {505- audioData.close();506- }507- },508- error: (error) => {509- disableAudioForSession(`Opus decoder failed: ${error}`, session);510- },511- });512- session.decoder.configure(AUDIO_DECODER_CONFIG);513-}514-515-function primeAudioOutputForActivation() {516- const session = activeAudioSession;517- const AudioContextConstructor = window.AudioContext || window.webkitAudioContext;518- if (typeof AudioContextConstructor !== 'function') return;519- if (!session.context || session.context.state === 'closed') {520- try {521- session.context = new AudioContextConstructor({ latencyHint: 'interactive', sampleRate: 48000 });522- } catch (_) {523- return;524- }525- }526- // This function is deliberately synchronous and is called before the first527- // await in a click/keyboard connect handler, preserving WebKit user activation.528- void session.context.resume().catch(() => {});529-}530-531-async function initializeAudioPipeline(session) {532- if (session.decoder) {533- try { session.decoder.close(); } catch (_) {}534- session.decoder = null;535- }536- if (session.workletNode) {537- try { session.workletNode.disconnect(); } catch (_) {}538- session.workletNode = null;539- }540- session.messageTracker.clear();541- if (typeof AudioDecoder !== 'function') {542- return { supported: false, error: 'WebCodecs AudioDecoder is unavailable' };543- }544- if (typeof AudioWorkletNode !== 'function') {545- return { supported: false, error: 'AudioWorklet is unavailable' };546- }547- let support;548- try {549- support = await AudioDecoder.isConfigSupported(AUDIO_DECODER_CONFIG);550- } catch (error) {551- return { supported: false, error: `Opus capability probe failed: ${error}` };552- }553- if (!support.supported) {554- return { supported: false, error: 'WebCodecs Opus decoding is unsupported' };555- }556- if (!session.context) {557- return { supported: false, error: 'Web Audio output is unavailable' };558- }559- try {560- if (session.context.sampleRate !== 48000) {561- throw new Error(`audio output opened at ${session.context.sampleRate} Hz instead of 48000 Hz`);562- }563- await session.context.audioWorklet.addModule(new URL('./audio-worklet.js', import.meta.url));564- session.workletNode = new AudioWorkletNode(session.context, 'sigil-audio-processor', {565- numberOfInputs: 0,566- numberOfOutputs: 1,567- outputChannelCount: [2],568- });569- session.workletNode.port.onmessage = (event) => {570- if (event.data?.type === 'accepted') {571- session.messageTracker.accept(event.data.id);572- if (session === activeAudioSession) updateAudioUI();573- } else if (event.data?.type === 'stats') {574- if (!isCurrentAvSyncEpoch(event.data.avSyncEpoch, session.avSyncEpoch)) return;575- session.workletDroppedFrames = Number.isSafeInteger(event.data.droppedFrames)576- && event.data.droppedFrames >= 0577- ? event.data.droppedFrames : session.workletDroppedFrames;578- session.workletRecoveryDiscardedFrames = Number.isSafeInteger(579- event.data.recoveryDiscardedFrames,580- ) && event.data.recoveryDiscardedFrames >= 0581- ? event.data.recoveryDiscardedFrames : session.workletRecoveryDiscardedFrames;582- session.audioTimeline = audioOutputTimelineFromStats(event.data);583- if (session !== activeAudioSession) return;584- session.bufferedFrames = Number.isSafeInteger(event.data.bufferedFrames)585- ? event.data.bufferedFrames : session.bufferedFrames;586- session.underflows = Number.isSafeInteger(event.data.underflows)587- ? event.data.underflows : session.underflows;588- session.underflowDurationMicros = Number.isFinite(event.data.underflowDurationMicros)589- && event.data.underflowDurationMicros >= 0590- ? event.data.underflowDurationMicros : session.underflowDurationMicros;591- session.silentDurationMicros = Number.isFinite(event.data.silentDurationMicros)592- && event.data.silentDurationMicros >= 0593- ? event.data.silentDurationMicros : session.silentDurationMicros;594- if (session.available && !session.muted) {595- setAudioState(event.data.started ? 'playing' : 'priming', 'bounded Opus playback');596- } else {597- updateAudioUI();598- }599- } else if (event.data?.type === 'error') {600- disableAudioForSession(`AudioWorklet failed: ${event.data.error}`, session);601- }602- };603- session.workletNode.port.onmessageerror = () => {604- disableAudioForSession('AudioWorklet returned an unreadable message', session);605- };606- session.workletNode.onprocessorerror = () => {607- disableAudioForSession('AudioWorklet processor stopped unexpectedly', session);608- };609- session.workletNode.connect(session.context.destination);610- session.workletNode.port.postMessage({ type: 'mute', muted: session.muted });611- configureAudioDecoder(session);612- await session.context.resume();613- setAudioState(614- session.context.state === 'running' ? 'negotiating' : 'blocked',615- session.context.state === 'running'616- ? 'Opus output ready'617- : 'WebKit suspended audio output; activate the audio button to retry',618- );619- return { supported: true, error: null };620- } catch (error) {621- await teardownAudioPipeline(false);622- return { supported: false, error: `Audio initialization failed: ${error}` };623- }624-}625-626-async function teardownAudioPipeline(resetStatus = true) {627- const session = activeAudioSession;628- videoPipeline.resetAvSync(session);629- const decoder = session.decoder;630- const workletNode = session.workletNode;631- const context = session.context;632- // Replace the active owner before closing resources so delayed callbacks633- // cannot mutate the disconnected UI or a successor session. Seed the next634- // dormant session with the user's mute preference across reconnects.635- activeAudioSession = newAudioSession({ muted: session.muted });636- session.expectedGeneration = null;637- session.channel = null;638- session.available = false;639- session.messageTracker.clear();640- session.decoder = null;641- session.workletNode = null;642- session.context = null;643- if (decoder) {644- try { decoder.close(); } catch (_) {}645- }646- if (workletNode) {647- try { workletNode.port.postMessage({ type: 'clear' }); } catch (_) {}648- try { workletNode.disconnect(); } catch (_) {}649- }650- if (context && context.state !== 'closed') {651- try { await context.close(); } catch (_) {}652- }653- if (resetStatus) updateAudioUI();654-}655-656430 async function toggleAudioMute() {657- const session = activeAudioSession;658- if (!session.available) return;659- session.muted = !session.muted;660- try {661- if (!session.muted && session.context?.state !== 'running') await session.context?.resume();662- session.workletNode?.port.postMessage({ type: 'mute', muted: session.muted });663- if (!session.muted && session.context?.state !== 'running') {664- setAudioState('blocked', 'WebKit suspended audio output; click the audio button to retry');665- } else {666- updateAudioUI();667- }668- } catch (error) {669- setAudioState('error', `Audio output activation failed: ${error}`);670- }671-}672-673-function prepareAudioForConnection() {674- activeAudioSession = newAudioSession({ muted: activeAudioSession.muted });675- activeAudioSession.acceptsNativeEvents = true;676- // Keep this synchronous and before createConnectionAttempt's first await so677- // WebKit observes the originating click/keyboard user activation.678- primeAudioOutputForActivation();431+ await audioPipeline.toggleMute();679432 }680433 681434 async function createConnectionAttempt({ createChannel: Channel }) {@@ -690,29 +443,26 @@ async function createConnectionAttempt({ createChannel: Channel }) {
690443 adaptiveDecision = null;691444 resetStreamTelemetry();692445 if (typeof Channel !== 'function') throw new Error('Tauri binary channels are unavailable');693- const audioSession = activeAudioSession;694- resetAudioTelemetry(audioSession);446+ const audioSession = audioPipeline.beginAttempt();695447 const frameSession = newFrameSession();696448 const pointerSession = newPointerSession();697449 activeFrameSession = frameSession;698450 activePointerSession = pointerSession;699- const audioSupport = await initializeAudioPipeline(audioSession);700- if (!audioSupport.supported) setAudioState('unavailable', audioSupport.error);451+ const audioAttempt = await audioPipeline.createAttemptChannel(Channel, audioSession);701452 activeFrameChannel = new Channel((message) => handleBinaryFrameMessage(message, frameSession));702- audioSession.channel = new Channel((message) => handleBinaryAudioMessage(message, audioSession));703453 pointerSession.channel = new Channel(704454 (message) => handlePointerPositionFeedback(message, pointerSession),705455 );706456 return {707- audioSession,457+ audioSession: audioAttempt.session,708458 frameSession,709459 pointerSession,710- audioSupport,460+ audioSupport: audioAttempt.support,711461 connectionArgs: {712462 frameChannel: activeFrameChannel,713- audioChannel: audioSession.channel,463+ audioChannel: audioAttempt.channel,714464 pointerChannel: pointerSession.channel,715- audioSupported: audioSupport.supported,465+ audioSupported: audioAttempt.support.supported,716466 },717467 };718468 }@@ -763,10 +513,8 @@ async function teardownConnectionAttempt(attempt) {
763513 const pointerSession = attempt?.pointerSession ?? activePointerSession;764514 if (activePointerSession === pointerSession) activePointerSession = null;765515 if (pointerSession) pointerSession.channel = null;766- if (attempt?.audioSession && attempt.audioSession !== activeAudioSession) {767- attempt.audioSession.channel = null;768- }769- await teardownAudioPipeline();516+ audioPipeline.releaseAttempt(attempt?.audioSession);517+ await audioPipeline.teardown();770518 }771519 772520 function updateAcceptedConnectionState({ attempt, state }) {@@ -783,32 +531,10 @@ function updateAcceptedConnectionState({ attempt, state }) {
783531 }784532 785533 async function showConnectedAttempt({ result, attempt }) {786- const { audioSession, audioSupport } = attempt;787- audioSession.expectedGeneration = Number.isSafeInteger(result.audio_generation)788- && result.audio_generation > 0789- ? result.audio_generation : null;790- const pendingAudioTerminal = takeAudioTerminalState(791- audioSession.pendingTerminalStates,792- audioSession.expectedGeneration,793- );794- if (pendingAudioTerminal !== null) {795- audioSession.failed = true;796- audioSession.failureDetail = pendingAudioTerminal.error || 'Audio connection ended';797- }798- audioSession.available = result.audio_available === true;799- if (audioSession.available && audioSession.failed) {800- disableAudioForSession(audioSession.failureDetail || 'Audio output failed', audioSession);801- } else if (audioSession.available) {802- setAudioState(803- audioSession.context?.state === 'running' ? 'priming' : 'blocked',804- audioSession.context?.state === 'running'805- ? 'Waiting for bounded Opus prebuffer'806- : 'WebKit suspended audio output; activate the audio button to retry',807- );808- } else {809- await teardownAudioPipeline(false);810- setAudioState('unavailable', result.audio_error || audioSupport.error || 'host audio unavailable');811- }534+ await audioPipeline.acceptConnectedResult(result, {535+ session: attempt.audioSession,536+ support: attempt.audioSupport,537+ });812538 updatePanelVisibility();813539 document.getElementById('node-id-text').textContent = result.host_node_id.substring(0, 16) + '...';814540 document.getElementById('frame-canvas').style.display = 'block';@@ -854,7 +580,7 @@ async function teardownConnectedResources() {
854580 const pointerSession = activePointerSession;855581 activePointerSession = null;856582 if (pointerSession) pointerSession.channel = null;857- await teardownAudioPipeline();583+ await audioPipeline.teardown();858584 videoPipeline.teardown();859585 }860586 @@ -886,7 +612,7 @@ async function showDisconnectedState() {
886612 const connectionState = newConnectionState({887613 invokeCommand: invoke,888614 createChannel: window.__TAURI__.core.Channel,889- beforeConnect: prepareAudioForConnection,615+ beforeConnect: audioPipeline.prepareForConnection,890616 createAttempt: createConnectionAttempt,891617 validateConnectedResult: validateConnectedAttempt,892618 activateAttempt: activateConnectedAttempt,@@ -982,54 +708,16 @@ function requestDecoderKeyframe(reason) {
982708 });983709 }984710 985-function resetAudioSyncTelemetry(session = activeAudioSession, { recoverRing = false } = {}) {986- if (!session) return;987- session.audioTimeline = null;988- session.avSyncEpoch = nextAvSyncEpoch(session.avSyncEpoch);989- try {990- session.workletNode?.port.postMessage({991- type: recoverRing ? 'recover' : 'av-sync-epoch',992- avSyncEpoch: session.avSyncEpoch,993- });994- } catch (_) {}995-}996-997-function sampleAvSkew(videoPtsMicros, presentationTimeMs) {998- const session = activeAudioSession;999- if (1000- exactMediaTimestampMicros(videoPtsMicros) === null1001- || !session1002- || session.failed1003- || !session.audioTimeline1004- || !session.available1005- || session.context?.state !== 'running'1006- || typeof session.context.getOutputTimestamp !== 'function'1007- ) return null;1008-1009- let outputTimestamp;1010- try {1011- outputTimestamp = session.context.getOutputTimestamp();1012- } catch (_) {1013- return null;1014- }1015- const audioPtsMicros = projectAudioMediaPtsMicros(1016- session.audioTimeline,1017- outputTimestamp,1018- presentationTimeMs,1019- );1020- return audioMinusVideoSkewMs(audioPtsMicros, videoPtsMicros);1021-}1022-1023711 const canvas = document.getElementById('frame-canvas');1024712 const remotePointer = document.getElementById('remote-pointer');1025713 const ctx = canvas.getContext('2d');1026-const videoPipeline = createVideoPipelineSession({714+videoPipeline = createVideoPipelineSession({1027715 hasWebCodecs,1028716 canvas,1029717 context: ctx,1030718 requestKeyframe: requestDecoderKeyframe,1031- resetAudioSync: resetAudioSyncTelemetry,1032- sampleAudioSkew: sampleAvSkew,719+ resetAudioSync: audioPipeline.resetSyncTelemetry,720+ sampleAudioSkew: audioPipeline.sampleSkew,1033721 onFormatChanged: () => {1034722 sizeCanvasToIncomingStream();1035723 if (connectionState.connected) void fitWindowToIncomingStream();@@ -1352,84 +1040,6 @@ function handleBinaryFrameMessage(message, session) {
13521040 }13531041 }13541042 1355-function acknowledgeAudio(generation, deliveryId) {1356- void invoke('iroh_client_ack_audio', { generation, deliveryId }).catch((error) => {1357- console.warn('audio acknowledgment failed:', error);1358- });1359-}1360-1361-function requestNativeAudioStop(session) {1362- const generation = session?.expectedGeneration;1363- if (session !== activeAudioSession1364- || !Number.isSafeInteger(generation)1365- || generation < 01366- || session.stopRequested) return;1367- session.stopRequested = true;1368- void invoke('iroh_client_stop_audio', { expectedGeneration: generation }).catch((error) => {1369- console.warn('audio stop failed:', error);1370- });1371-}1372-1373-function disableAudioForSession(detail, session = activeAudioSession) {1374- if (!session || session !== activeAudioSession) return;1375- videoPipeline.resetAvSync(session);1376- session.failed = true;1377- session.failureDetail = detail;1378- session.available = false;1379- if (session.decoder) {1380- try { session.decoder.close(); } catch (_) {}1381- session.decoder = null;1382- }1383- session.messageTracker.clear();1384- try { session.workletNode?.port.postMessage({ type: 'clear' }); } catch (_) {}1385- requestNativeAudioStop(session);1386- setAudioState('error', detail);1387-}1388-1389-function handleBinaryAudioMessage(message, session) {1390- let packet;1391- try {1392- packet = parseAudioEnvelope(message);1393- } catch (error) {1394- console.error('invalid audio packet:', error);1395- disableAudioForSession(`Audio packet failed: ${error}`, session);1396- return;1397- }1398-1399- // Every valid delivery is released using the token embedded in that exact1400- // message. A delayed delivery from an old connection is acknowledged but1401- // cannot enter the current decoder.1402- acknowledgeAudio(packet.generation, packet.deliveryId);1403- if (session !== activeAudioSession1404- || session.failed1405- || !isCurrentAudioDelivery(packet, session.expectedGeneration)) return;1406-1407- try {1408- session.packetsReceived++;1409- if (!session.decoder || session.decoder.state === 'closed') {1410- session.decoderDropped++;1411- updateAudioUI();1412- return;1413- }1414- const queuedAudioPackets = session.decoder.decodeQueueSize;1415- if (packet.discontinuity || queuedAudioPackets >= MAX_AUDIO_DECODE_QUEUE_SIZE) {1416- videoPipeline.resetAvSync(session, { recoverRing: true });1417- session.decoderDropped += queuedAudioPackets;1418- configureAudioDecoder(session);1419- }1420- session.decoder.decode(new EncodedAudioChunk({1421- type: 'key',1422- timestamp: packet.ptsMicros,1423- duration: (packet.frameSamples * 1_000_000) / packet.sampleRate,1424- data: packet.data,1425- }));1426- updateAudioUI();1427- } catch (error) {1428- console.error('undecodable audio packet:', error);1429- disableAudioForSession(`Audio packet failed: ${error}`, session);1430- }1431-}1432-14331043 // The software decoder/JPEG compatibility path intentionally remains an event:14341044 // it is only selected when WebCodecs is unavailable and is not latency-critical.14351045 listen('frame', (event) => {@@ -1519,30 +1129,11 @@ listen('frame-error', (event) => {
15191129 });15201130 15211131 listen('audio-state', (event) => {1522- const session = activeAudioSession;1523- if (connectionState.disconnecting1524- || !session.acceptsNativeEvents1525- || event.payload?.available !== false) return;1526- if (session.expectedGeneration === null) {1527- try {1528- stageAudioTerminalState(session.pendingTerminalStates, event.payload);1529- } catch (error) {1530- console.error('invalid pre-connect audio terminal state:', error);1531- }1532- return;1533- }1534- if (!isCurrentAudioGeneration(event.payload?.generation, session.expectedGeneration)) return;1535- disableAudioForSession(event.payload.error || 'Audio connection ended', session);1132+ audioPipeline.handleNativeState(event.payload);15361133 });15371134 15381135 listen('audio-stats', (event) => {1539- const session = activeAudioSession;1540- if (!isCurrentAudioGeneration(event.payload?.generation, session.expectedGeneration)) return;1541- session.transportDropped = Number.isSafeInteger(event.payload?.sequence_dropped_total)1542- ? event.payload.sequence_dropped_total : session.transportDropped;1543- session.frontendDropped = Number.isSafeInteger(event.payload?.frontend_dropped_total)1544- ? event.payload.frontend_dropped_total : session.frontendDropped;1545- updateAudioUI();1136+ audioPipeline.handleNativeStats(event.payload);15461137 });15471138 15481139 // ─── Input ────────────────────────────────────────────────────────────────────